initial fleet support

config.example.json

@@ -7,6 +7,8 @@
   "twitter_consumer_secret": "",
   "twitter_access_token_key": "",
   "twitter_access_token_secret": "",
+  "twitter_private_auth_token": "",
+  "twitter_private_csrf_token": "",
   "mode": 0,
   "resume_on_start": false,
   "work_interval": 60,

dist/command.js

@@ -1,12 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.query = exports.view = exports.unsub = exports.list = exports.sub = exports.parseCmd = void 0;
+exports.unsub = exports.list = exports.sub = exports.parseCmd = void 0;
 const fs = require("fs");
 const path = require("path");
 const datetime_1 = require("./datetime");
 const loggers_1 = require("./loggers");
 const twitter_1 = require("./twitter");
-const utils_1 = require("./utils");
 const logger = loggers_1.getLogger('command');
 function parseCmd(message) {
     message = message.trim();
@@ -29,17 +28,8 @@ function parseCmd(message) {
 }
 exports.parseCmd = parseCmd;
 function parseLink(link) {
-    let match = link.match(/twitter.com\/([^\/?#]+)\/lists\/([^\/?#]+)/) ||
-        link.match(/^([^\/?#]+)\/([^\/?#]+)$/);
-    if (match)
-        return [match[1], `/lists/${match[2]}`];
-    match =
-        link.match(/twitter.com\/([^\/?#]+)\/status\/(\d+)/);
-    if (match)
-        return [match[1], `/status/${match[2]}`];
-    match =
-        link.match(/twitter.com\/([^\/?#]+)/) ||
-            link.match(/^([^\/?#]+)$/);
+    let match = link.match(/twitter.com\/([^\/?#]+)/) ||
+        link.match(/^([^\/?#]+)$/);
     if (match)
         return [match[1]];
     return;
@@ -63,36 +53,26 @@ function sub(chat, args, reply, lock, lockfile) {
         return reply('请先添加机器人为好友。');
     }
     if (args.length === 0) {
-        return reply('找不到要订阅的链接。');
+        return reply('找不到要订阅推特故事的链接。');
     }
     const match = parseLink(args[0]);
     if (!match) {
         return reply(`订阅链接格式错误：
-示例：
-https://twitter.com/Saito_Shuka
-https://twitter.com/rikakomoe/lists/lovelive
-https://twitter.com/TomoyoKurosawa/status/1294613494860361729`);
-    }
-    let offset = '0';
-    if (match[1]) {
-        const matchStatus = match[1].match(/\/status\/(\d+)/);
-        if (matchStatus) {
-            offset = utils_1.BigNumOps.plus(matchStatus[1], '-1');
-            delete match[1];
-        }
+示例：https://twitter.com/sunflower930316`);
     }
     const subscribeTo = (link, config = {}) => {
-        const { addNew = false, msg = `已为此聊天订阅 ${link}` } = config;
+        const { addNew = false, msg = `已为此聊天订阅 ${link} 的推特故事` } = config;
         if (addNew) {
             lock.feed.push(link);
             lock.threads[link] = {
-                offset,
+                permaFeed: twitter_1.ScreenNameNormalizer.permaFeeds[link],
+                offset: '0',
                 subscribers: [],
                 updatedAt: '',
             };
         }
         lock.threads[link].subscribers.push(chat);
-        logger.warn(`chat ${JSON.stringify(chat)} has subscribed ${link}`);
+        logger.warn(`chat ${JSON.stringify(chat)} has subscribed fleets for ${link}`);
         fs.writeFileSync(path.resolve(lockfile), JSON.stringify(lock));
         reply(msg);
     };
@@ -102,18 +82,10 @@ https://twitter.com/TomoyoKurosawa/status/1294613494860361729`);
     if (realLink)
         return subscribeTo(realLink);
     const [rawUserName, more] = match;
-    if (rawUserName.toLowerCase() === 'i' && (more === null || more === void 0 ? void 0 : more.match(/lists\/(\d+)/))) {
-        return subscribeTo(linkBuilder('i', more), { addNew: true });
-    }
     twitter_1.ScreenNameNormalizer.normalizeLive(rawUserName).then(userName => {
         if (!userName)
             return reply(`找不到用户 ${rawUserName.replace(/^@?(.*)$/, '@$1')}。`);
-        const link = linkBuilder(userName, more);
-        const msg = (offset === '0') ?
-            undefined :
-            `已为此聊天订阅 ${link} 并回溯到此动态 ID（含）之后的第一条动态。
-（参见：https://blog.twitter.com/engineering/en_us/a/2010/announcing-snowflake.html）`;
-        subscribeTo(link, { addNew: true, msg });
+        subscribeTo(linkBuilder(userName, more), { addNew: true });
     });
 }
 exports.sub = sub;
@@ -122,7 +94,7 @@ function unsub(chat, args, reply, lock, lockfile) {
         return reply('请先添加机器人为好友。');
     }
     if (args.length === 0) {
-        return reply('找不到要退订的链接。');
+        return reply('找不到要退订推特故事的链接。');
     }
     const match = parseLink(args[0]);
     if (!match) {
@@ -130,12 +102,12 @@ function unsub(chat, args, reply, lock, lockfile) {
     }
     const [link, index] = linkFinder(match, chat, lock);
     if (index === -1)
-        return list(chat, args, msg => reply('您没有订阅此链接。\n' + msg), lock);
+        return list(chat, args, msg => reply('您没有订阅此链接的推特故事。\n' + msg), lock);
     else {
         lock.threads[link].subscribers.splice(index, 1);
         fs.writeFileSync(path.resolve(lockfile), JSON.stringify(lock));
         logger.warn(`chat ${JSON.stringify(chat)} has unsubscribed ${link}`);
-        return reply(`已为此聊天退订 ${link}`);
+        return reply(`已为此聊天退订 ${link} 的推特故事`);
     }
 }
 exports.unsub = unsub;
@@ -148,62 +120,6 @@ function list(chat, _, reply, lock) {
         if (lock.threads[key].subscribers.find(({ chatID, chatType }) => chat.chatID === chatID && chat.chatType === chatType))
             links.push(`${key} ${datetime_1.relativeDate(lock.threads[key].updatedAt)}`);
     });
-    return reply('此聊天中订阅的链接：\n' + links.join('\n'));
+    return reply('此聊天中订阅推特故事的链接：\n' + links.join('\n'));
 }
 exports.list = list;
-function view(chat, args, reply) {
-    if (args.length === 0) {
-        return reply('找不到要查看的链接。');
-    }
-    const match = args[0].match(/^(?:.*twitter.com\/[^\/?#]+\/status\/)?(\d+)/);
-    if (!match) {
-        return reply('链接格式有误。');
-    }
-    try {
-        twitter_1.sendTweet(match[1], chat);
-    }
-    catch (e) {
-        reply('推特机器人尚未加载完毕，请稍后重试。');
-    }
-}
-exports.view = view;
-function query(chat, args, reply) {
-    if (args.length === 0) {
-        return reply('找不到要查询的用户。');
-    }
-    const match = args[0].match(/twitter.com\/([^\/?#]+)/) ||
-        args[0].match(/^([^\/?#]+)$/);
-    if (!match) {
-        return reply('链接格式有误。');
-    }
-    const conf = { username: match[1], noreps: 'on', norts: 'off' };
-    const confZH = {
-        count: '数量上限',
-        since: '起始点',
-        until: '结束点',
-        noreps: '忽略回复推文（on/off）',
-        norts: '忽略原生转推（on/off）',
-    };
-    for (const arg of args.slice(1)) {
-        const optMatch = arg.match(/^(count|since|until|noreps|norts)=(.*)/);
-        if (!optMatch)
-            return reply(`未定义的查询参数：${arg}。`);
-        const optKey = optMatch[1];
-        if (optMatch.length === 1)
-            return reply(`查询${confZH[optKey]}参数格式有误。`);
-        conf[optKey] = optMatch[2];
-        if (optMatch[2] === '')
-            return reply(`查询${confZH[optKey]}参数值不可为空。`);
-    }
-    if (conf.count !== undefined && !Number(conf.count) || Math.abs(Number(conf.count)) > 50) {
-        return reply('查询数量上限参数为零、非数值或超出取值范围。');
-    }
-    try {
-        twitter_1.sendTimeline(conf, chat);
-    }
-    catch (e) {
-        logger.error(`error querying timeline, error: ${e}`);
-        reply('推特机器人尚未加载完毕，请稍后重试。');
-    }
-}
-exports.query = query;

dist/helper.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+function default_1(message) {
+    message = message.trim();
+    message = message.replace('\\\\', '\\0x5c');
+    message = message.replace('\\\"', '\\0x22');
+    message = message.replace('\\\'', '\\0x27');
+    const strs = message.match(/'[\s\S]*?'|"[\s\S]*?"|\S*\[CQ:[\s\S]*?\]\S*|\S+/mg);
+    const cmd = (strs === null || strs === void 0 ? void 0 : strs.length) ? strs[0].length ? strs[0].substring(0, 1) === '/' ? strs[0].substring(1) : '' : '' : '';
+    const args = strs === null || strs === void 0 ? void 0 : strs.slice(1).map(arg => {
+        arg = arg.replace(/^["']+|["']+$/g, '');
+        arg = arg.replace('\\0x27', '\\\'');
+        arg = arg.replace('\\0x22', '\\\"');
+        arg = arg.replace('\\0x5c', '\\\\');
+        return arg;
+    });
+    return {
+        cmd,
+        args,
+    };
+}
+exports.default = default_1;

dist/main.js

@@ -136,6 +136,8 @@ const worker = new twitter_1.default({
     consumer_secret: config.twitter_consumer_secret,
     access_token_key: config.twitter_access_token_key,
     access_token_secret: config.twitter_access_token_secret,
+    private_auth_token: config.twitter_private_auth_token,
+    private_csrf_token: config.twitter_private_csrf_token,
     lock,
     lockfile: config.lockfile,
     workInterval: config.work_interval,

dist/mirai.js

@@ -124,13 +124,13 @@ class default_1 {
                 host: this.botInfo.host,
                 port: this.botInfo.port,
             });
-            this.bot.axios.defaults.maxContentLength = Infinity;
+            this.bot.axios.defaults.maxContentLength = this.bot.axios.defaults.maxBodyLength = Infinity;
             this.bot.on('NewFriendRequestEvent', evt => {
                 logger.debug(`detected new friend request event: ${JSON.stringify(evt)}`);
                 this.bot.api.groupList()
                     .then((groupList) => {
                     if (groupList.some(groupItem => groupItem.id === evt.groupId)) {
-                        evt.respond('allow');
+                        evt.respond(0); // allow
                         return logger.info(`accepted friend request from ${evt.fromId} (from group ${evt.groupId})`);
                     }
                     logger.warn(`received friend request from ${evt.fromId} (from group ${evt.groupId})`);
@@ -142,7 +142,7 @@ class default_1 {
                 this.bot.api.friendList()
                     .then((friendList) => {
                     if (friendList.some(friendItem => friendItem.id = evt.fromId)) {
-                        evt.respond('allow');
+                        evt.respond(0); // allow
                         return logger.info(`accepted group invitation from ${evt.fromId} (friend)`);
                     }
                     logger.warn(`received group invitation from ${evt.fromId} (unknown)`);
@@ -153,57 +153,23 @@ class default_1 {
                 const chat = yield this.getChat(msg);
                 const cmdObj = command_1.parseCmd(msg.plain);
                 switch (cmdObj.cmd) {
-                    case 'twitter_view':
-                    case 'twitter_get':
-                        command_1.view(chat, cmdObj.args, msg.reply);
-                        break;
-                    case 'twitter_query':
-                    case 'twitter_gettimeline':
-                        command_1.query(chat, cmdObj.args, msg.reply);
-                        break;
-                    case 'twitter_sub':
-                    case 'twitter_subscribe':
+                    case 'twitterfleets_sub':
+                    case 'twitterfleets_subscribe':
                         this.botInfo.sub(chat, cmdObj.args, msg.reply);
                         break;
-                    case 'twitter_unsub':
-                    case 'twitter_unsubscribe':
+                    case 'twitterfleets_unsub':
+                    case 'twitterfleets_unsubscribe':
                         this.botInfo.unsub(chat, cmdObj.args, msg.reply);
                         break;
                     case 'ping':
-                    case 'twitter':
+                    case 'twitterfleets':
                         this.botInfo.list(chat, cmdObj.args, msg.reply);
                         break;
                     case 'help':
-                        if (cmdObj.args.length === 0) {
-                            msg.reply(`推特搬运机器人：
-/twitter - 查询当前聊天中的推文订阅
-/twitter_subscribe〈链接|用户名〉- 订阅 Twitter 推文搬运
-/twitter_unsubscribe〈链接|用户名〉- 退订 Twitter 推文搬运
-/twitter_view〈链接〉- 查看推文
-/twitter_query〈链接|用户名〉[参数列表...] - 查询时间线（详见 /help twitter_query）\
-${chat.chatType === "temp" /* Temp */ ?
-                                '\n（当前游客模式下无法使用订阅功能，请先添加本账号为好友。）' : ''}`);
-                        }
-                        else if (cmdObj.args[0] === 'twitter_query') {
-                            msg.reply(`查询时间线中的推文：
-/twitter_query〈链接|用户名〉[〈参数 1〉=〈值 1〉〈参数 2〉=〈值 2〉...]
-
-参数列表（方框内全部为可选，留空则为默认）：
-    count：查询数量上限（类型：非零整数，最大值正负 50）[默认值：10]
-    since：查询起始点（类型：正整数或日期）[默认值：（空，无限过去）]
-    until：查询结束点（类型：正整数或日期）[默认值：（空，当前时刻）]
-    noreps 忽略回复推文（类型：on/off）[默认值：on（是）]
-    norts：忽略原生转推（类型：on/off）[默认值：off（否）]`)
-                                .then(() => msg.reply(`\
-起始点和结束点为正整数时取推特推文编号作为比较基准，否则会尝试作为日期读取。
-推荐的日期格式：2012-12-22 12:22 UTC+2 （日期和时间均为可选，可分别添加）
-count 为正时，从新向旧查询；为负时，从旧向新查询
-count 与 since/until 并用时，取二者中实际查询结果较少者
-例子：/twitter_query RiccaTachibana count=5 since="2019-12-30\
- UTC+9" until="2020-01-06 UTC+8" norts=on
-    从起始时间点（含）到结束时间点（不含）从新到旧获取最多 5 条推文，\
-其中不包含原生转推（实际上用户只发了 1 条）`));
-                        }
+                        msg.reply(`推特故事搬运机器人：
+/twitterfleets - 查询当前聊天中的推特故事订阅
+/twitterfleets_subscribe [链接] - 订阅 Twitter Fleets 搬运
+/twitterfleets_unsubscribe [链接] - 退订 Twitter Fleets 搬运`);
                 }
             }));
         };

dist/twitter.js

@@ -9,14 +9,18 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sendTimeline = exports.sendTweet = exports.ScreenNameNormalizer = void 0;
+exports.ScreenNameNormalizer = void 0;
 const fs = require("fs");
 const path = require("path");
+const request = require("request");
 const Twitter = require("twitter");
 const loggers_1 = require("./loggers");
 const utils_1 = require("./utils");
 const webshot_1 = require("./webshot");
 class ScreenNameNormalizer {
+    static savePermaFeedForUser(user) {
+        this.permaFeeds[`https://twitter.com/${user.screen_name}`] = `https://twitter.com/i/user/${user.id_str}`;
+    }
     static normalizeLive(username) {
         return __awaiter(this, void 0, void 0, function* () {
             if (this._queryUser) {
@@ -34,16 +38,8 @@ class ScreenNameNormalizer {
     }
 }
 exports.ScreenNameNormalizer = ScreenNameNormalizer;
+ScreenNameNormalizer.permaFeeds = {};
 ScreenNameNormalizer.normalize = (username) => username.toLowerCase().replace(/^@/, '');
-exports.sendTweet = (id, receiver) => {
-    throw Error();
-};
-exports.sendTimeline = (conf, receiver) => {
-    throw Error();
-};
-const TWITTER_EPOCH = 1288834974657;
-const snowflake = (epoch) => Number.isNaN(epoch) ? undefined :
-    utils_1.BigNumOps.lShift(String(epoch - 1 - TWITTER_EPOCH), 22);
 const logger = loggers_1.getLogger('twitter');
 const maxTrials = 3;
 const uploadTimeout = 10000;
@@ -77,55 +73,11 @@ class default_1 {
             this.webshot = new webshot_1.default(this.mode, () => setTimeout(this.work, this.workInterval * 1000));
         };
         this.queryUser = (username) => this.client.get('users/show', { screen_name: username })
-            .then((user) => user.screen_name);
-        this.queryTimelineReverse = (conf) => {
-            if (!conf.since)
-                return this.queryTimeline(conf);
-            const count = conf.count;
-            const maxID = conf.until;
-            conf.count = undefined;
-            const until = () => utils_1.BigNumOps.min(maxID, utils_1.BigNumOps.plus(conf.since, String(7 * 24 * 3600 * 1000 * Math.pow(2, 22))));
-            conf.until = until();
-            const promise = (tweets) => this.queryTimeline(conf).then(newTweets => {
-                tweets = newTweets.concat(tweets);
-                conf.since = conf.until;
-                conf.until = until();
-                if (tweets.length >= count ||
-                    utils_1.BigNumOps.compare(conf.since, conf.until) >= 0) {
-                    return tweets.slice(-count);
-                }
-                return promise(tweets);
-            });
-            return promise([]);
-        };
-        this.queryTimeline = ({ username, count, since, until, noreps, norts }) => {
-            username = username.replace(/^@?(.*)$/, '@$1');
-            logger.info(`querying timeline of ${username} with config: ${JSON.stringify(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, (count && { count })), (since && { since })), (until && { until })), (noreps && { noreps })), (norts && { norts })))}`);
-            const fetchTimeline = (config = {
-                screen_name: username.slice(1),
-                trim_user: true,
-                exclude_replies: noreps !== null && noreps !== void 0 ? noreps : true,
-                include_rts: !(norts !== null && norts !== void 0 ? norts : false),
-                since_id: since,
-                max_id: until,
-                tweet_mode: 'extended',
-            }, tweets = []) => this.client.get('statuses/user_timeline', config)
-                .then((newTweets) => {
-                if (newTweets.length) {
-                    logger.debug(`fetched tweets: ${JSON.stringify(newTweets)}`);
-                    config.max_id = utils_1.BigNumOps.plus('-1', newTweets[newTweets.length - 1].id_str);
-                    logger.info(`timeline query of ${username} yielded ${newTweets.length} new tweets, next query will start at offset ${config.max_id}`);
-                    tweets.push(...newTweets);
-                }
-                if (!newTweets.length || tweets.length >= count) {
-                    logger.info(`timeline query of ${username} finished successfully, ${tweets.length} tweets have been fetched`);
-                    return tweets.slice(0, count);
-                }
-                return fetchTimeline(config, tweets);
-            });
-            return fetchTimeline();
-        };
-        this.workOnTweets = (tweets, sendTweets) => {
+            .then((user) => {
+            ScreenNameNormalizer.savePermaFeedForUser(user);
+            return user.screen_name;
+        });
+        this.workOnFleets = (user, fleets, sendFleets) => {
             const uploader = (message, lastResort) => {
                 let timeout = uploadTimeout;
                 return retryOnError(() => this.bot.uploadPic(message, timeout).then(() => message), (_, count, terminate) => {
@@ -139,21 +91,9 @@ class default_1 {
                     }
                 });
             };
-            return this.webshot(tweets, uploader, sendTweets, this.webshotDelay);
+            return this.webshot(user, fleets, uploader, sendFleets, this.webshotDelay);
         };
-        this.getTweet = (id, sender) => {
-            const endpoint = 'statuses/show';
-            const config = {
-                id,
-                tweet_mode: 'extended',
-            };
-            return this.client.get(endpoint, config)
-                .then((tweet) => {
-                logger.debug(`api returned tweet ${JSON.stringify(tweet)} for query id=${id}`);
-                return this.workOnTweets([tweet], sender);
-            });
-        };
-        this.sendTweets = (source, ...to) => (msg, text, author) => {
+        this.sendFleets = (source, ...to) => (msg, text) => {
             to.forEach(subscriber => {
                 logger.info(`pushing data${source ? ` of ${source}` : ''} to ${JSON.stringify(subscriber)}`);
                 retryOnError(() => this.bot.sendTo(subscriber, msg), (_, count, terminate) => {
@@ -163,7 +103,7 @@ class default_1 {
                     else {
                         logger.warn(`${count - 1} consecutive failures while sending` +
                             'message chain, trying plain text instead...');
-                        terminate(this.bot.sendTo(subscriber, author + text));
+                        terminate(this.bot.sendTo(subscriber, text));
                     }
                 });
             });
@@ -192,77 +132,53 @@ class default_1 {
             }
             const currentFeed = lock.feed[lock.workon];
             logger.debug(`pulling feed ${currentFeed}`);
-            const promise = new Promise(resolve => {
-                let match = currentFeed.match(/https:\/\/twitter.com\/([^\/]+)\/lists\/([^\/]+)/);
-                let config;
-                let endpoint;
-                if (match) {
-                    if (match[1] === 'i') {
-                        config = {
-                            list_id: match[2],
-                            tweet_mode: 'extended',
-                        };
-                    }
-                    else {
-                        config = {
-                            owner_screen_name: match[1],
-                            slug: match[2],
-                            tweet_mode: 'extended',
-                        };
-                    }
-                    endpoint = 'lists/statuses';
-                }
-                else {
-                    match = currentFeed.match(/https:\/\/twitter.com\/([^\/]+)/);
-                    if (match) {
-                        config = {
-                            screen_name: match[1],
-                            exclude_replies: false,
-                            tweet_mode: 'extended',
-                        };
-                        endpoint = 'statuses/user_timeline';
-                    }
-                }
-                if (endpoint) {
-                    const offset = lock.threads[currentFeed].offset;
-                    if (offset > 0)
-                        config.since_id = offset;
-                    this.client.get(endpoint, config, (error, tweets, response) => {
-                        if (error) {
-                            if (error instanceof Array && error.length > 0 && error[0].code === 34) {
-                                logger.warn(`error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
-                                lock.threads[currentFeed].subscribers.forEach(subscriber => {
-                                    logger.info(`sending notfound message of ${currentFeed} to ${JSON.stringify(subscriber)}`);
-                                    this.bot.sendTo(subscriber, `链接 ${currentFeed} 指向的用户或列表不存在，请退订。`).catch();
-                                });
-                            }
-                            else {
-                                logger.error(`unhandled error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
-                            }
-                            resolve();
-                        }
-                        else
-                            resolve(tweets);
-                    });
-                }
+            let user;
+            let match = currentFeed.match(/https:\/\/twitter.com\/([^\/]+)/);
+            if (match)
+                match = lock.threads[currentFeed].permaFeed.match(/https:\/\/twitter.com\/i\/user\/([^\/]+)/);
+            if (!match) {
+                logger.error(`cannot get endpoint for feed ${currentFeed}`);
+                return;
+            }
+            let endpoint = `https://api.twitter.com/fleets/v1/user_fleets?user_id=${match[1]}`;
+            const promise = new Promise((resolve, reject) => {
+                this.privateClient.get(endpoint, (error, fleetFeed, _) => {
+                    if (error)
+                        reject(error);
+                    else
+                        resolve(fleetFeed);
+                });
             });
-            promise.then((tweets) => {
-                logger.debug(`api returned ${JSON.stringify(tweets)} for feed ${currentFeed}`);
+            this.client.get('users/show', { user_id: match[1] })
+                .then((fullUser) => { user = fullUser; return promise; })
+                .catch(error => {
+                logger.error(`unhandled error on fetching fleets for ${currentFeed}: ${JSON.stringify(error)}`);
+            })
+                .then((fleetFeed) => {
+                logger.debug(`private api returned ${JSON.stringify(fleetFeed)} for feed ${currentFeed}`);
+                logger.debug(`api returned ${JSON.stringify(user)} for owner of feed ${currentFeed}`);
                 const currentThread = lock.threads[currentFeed];
                 const updateDate = () => currentThread.updatedAt = new Date().toString();
-                if (!tweets || tweets.length === 0) {
+                if (!fleetFeed || fleetFeed.fleet_threads.length === 0) {
                     updateDate();
                     return;
                 }
-                const topOfFeed = tweets[0].id_str;
-                const updateOffset = () => currentThread.offset = topOfFeed;
+                let fleets = fleetFeed.fleet_threads[0].fleets;
+                const bottomOfFeed = fleets.slice(-1)[0].fleet_id.substring(3);
+                const updateOffset = () => currentThread.offset = bottomOfFeed;
                 if (currentThread.offset === '-1') {
                     updateOffset();
                     return;
                 }
-                if (currentThread.offset === '0')
-                    tweets.splice(1);
-                return this.workOnTweets(tweets, this.sendTweets(`thread ${currentFeed}`, ...currentThread.subscribers))
+                if (currentThread.offset !== '0') {
+                    const readCount = fleets.findIndex(fleet => {
+                        return Number(utils_1.BigNumOps.plus(fleet.fleet_id.substring(3), `-${currentThread.offset}`)) > 0;
+                    });
+                    if (readCount === -1)
+                        return;
+                    fleets = fleets.slice(readCount);
+                }
+                return this.workOnFleets(user, fleets, this.sendFleets(`thread ${currentFeed}`, ...currentThread.subscribers))
                     .then(updateDate).then(updateOffset);
             })
                 .then(() => {
@@ -282,50 +198,18 @@ class default_1 {
             access_token_key: opt.access_token_key,
             access_token_secret: opt.access_token_secret,
         });
+        this.privateClient = new Twitter({
+            bearer_token: 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
+        });
+        this.privateClient.request = request.defaults({
+            headers: Object.assign(Object.assign({}, this.privateClient.options.request_options.headers), { 'Content-Type': 'application/x-www-form-urlencoded', 'Cookie': `auth_token=${opt.private_auth_token}; ct0=${opt.private_csrf_token};`, 'X-CSRF-Token': opt.private_csrf_token })
+        });
         this.lockfile = opt.lockfile;
         this.lock = opt.lock;
         this.workInterval = opt.workInterval;
         this.bot = opt.bot;
-        this.webshotDelay = opt.webshotDelay;
         this.mode = opt.mode;
         ScreenNameNormalizer._queryUser = this.queryUser;
-        exports.sendTweet = (id, receiver) => {
-            this.getTweet(id, this.sendTweets(`tweet ${id}`, receiver))
-                .catch((err) => {
-                if (err[0].code !== 144) {
-                    logger.warn(`error retrieving tweet: ${err[0].message}`);
-                    this.bot.sendTo(receiver, `获取推文时出现错误：${err[0].message}`);
-                }
-                this.bot.sendTo(receiver, '找不到请求的推文，它可能已被删除。');
-            });
-        };
-        exports.sendTimeline = ({ username, count, since, until, noreps, norts }, receiver) => {
-            const countNum = Number(count) || 10;
-            (countNum > 0 ? this.queryTimeline : this.queryTimelineReverse)({
-                username,
-                count: Math.abs(countNum),
-                since: utils_1.BigNumOps.parse(since) || snowflake(new Date(since).getTime()),
-                until: utils_1.BigNumOps.parse(until) || snowflake(new Date(until).getTime()),
-                noreps: { on: true, off: false }[noreps],
-                norts: { on: true, off: false }[norts],
-            })
-                .then(tweets => utils_1.chainPromises(tweets.map(tweet => this.bot.sendTo(receiver, `\
-编号：${tweet.id_str}
-时间：${tweet.created_at}
-媒体：${tweet.extended_entities ? '有' : '无'}
-正文：\n${tweet.full_text.replace(/^([\s\S\n]{50})[\s\S\n]+?( https:\/\/t.co\/.*)?$/, '$1…$2')}`))
-                .concat(this.bot.sendTo(receiver, tweets.length ?
-                '时间线查询完毕，使用 /twitter_view <编号> 查看推文详细内容。' :
-                '时间线查询完毕，没有找到符合条件的推文。'))))
-                .catch((err) => {
-                var _a, _b, _c;
-                if (((_a = err[0]) === null || _a === void 0 ? void 0 : _a.code) !== 34) {
-                    logger.warn(`error retrieving timeline: ${((_b = err[0]) === null || _b === void 0 ? void 0 : _b.message) || err}`);
-                    return this.bot.sendTo(receiver, `获取时间线时出现错误：${((_c = err[0]) === null || _c === void 0 ? void 0 : _c.message) || err}`);
-                }
-                this.bot.sendTo(receiver, `找不到用户 ${username.replace(/^@?(.*)$/, '@$1')}。`);
-            });
-        };
     }
 }
 exports.default = default_1;

dist/twitter_test.js

@@ -1,29 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const path = require("path");
-const twitter_1 = require("./twitter");
-const webshot_1 = require("./webshot");
-const configPath = './config.json';
-let worker;
-try {
-    const config = require(path.resolve(configPath));
-    worker = new twitter_1.default(Object.fromEntries(Object.entries(config).map(([k, v]) => [k.replace('twitter_', ''), v])));
-}
-catch (e) {
-    console.log('Failed to parse config file: ', configPath);
-    process.exit(1);
-}
-const webshot = new webshot_1.default(worker.mode, () => {
-    worker.webshot = webshot;
-    worker.getTweet('1296935552848035840', (msg, text, author) => {
-        console.log(author + text);
-        console.log(JSON.stringify(msg));
-    }).catch(console.log);
-    worker.getTweet('1296935552848035841', (msg, text, author) => {
-        console.log(author + text);
-        console.log(JSON.stringify(msg));
-    }).catch(console.log);
-});
-worker.queryUser('tomoyokurosawa').then(console.log).catch(console.log);
-worker.queryUser('tomoyourosawa').then(console.log).catch(console.log);
-worker.queryUser('@tomoyokurosawa').then(console.log).catch(console.log);

dist/utils.js

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BigNumOps = exports.chainPromises = void 0;
-exports.chainPromises = (promises, reducer = (p1, p2) => p1.then(() => p2), initialValue) => promises.reduce(reducer, Promise.resolve(initialValue));
+const chainPromises = (promises, reducer = (p1, p2) => p1.then(() => p2), initialValue) => promises.reduce(reducer, Promise.resolve(initialValue));
+exports.chainPromises = chainPromises;
 const splitBigNumAt = (num, at) => num.replace(RegExp(String.raw `^([+-]?)(\d+)(\d{${at}})$`), '$1$2,$1$3')
     .replace(/^([^,]*)$/, '0,$1').split(',')
     .map(Number);

dist/webshot.js

@@ -12,14 +12,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const axios_1 = require("axios");
 const CallableInstance = require("callable-instance");
 const html_entities_1 = require("html-entities");
-const pngjs_1 = require("pngjs");
-const puppeteer = require("puppeteer");
-const sharp = require("sharp");
-const util_1 = require("util");
 const gifski_1 = require("./gifski");
 const loggers_1 = require("./loggers");
 const mirai_1 = require("./mirai");
-const utils_1 = require("./utils");
 const xmlEntities = new html_entities_1.XmlEntities();
 const ZHType = (type) => new class extends String {
     constructor() {
@@ -37,199 +32,6 @@ const logger = loggers_1.getLogger('webshot');
 class Webshot extends CallableInstance {
     constructor(mode, onready) {
         super('webshot');
-        // use local Chromium
-        this.connect = (onready) => puppeteer.connect({ browserURL: 'http://127.0.0.1:9222' })
-            .then(browser => this.browser = browser)
-            .then(() => {
-            logger.info('launched puppeteer browser');
-            if (onready)
-                return onready();
-        })
-            .catch(error => this.reconnect(error, onready));
-        this.reconnect = (error, onready) => {
-            logger.error(`connection error, reason: ${error}`);
-            logger.warn('trying to reconnect in 2.5s...');
-            return util_1.promisify(setTimeout)(2500)
-                .then(() => this.connect(onready));
-        };
-        this.extendEntity = (media) => {
-            logger.info('not working on a tweet');
-        };
-        this.renderWebshot = (url, height, webshotDelay) => {
-            const jpeg = (data) => data.pipe(sharp()).jpeg({ quality: 90, trellisQuantisation: true });
-            const sharpToBase64 = (pic) => new Promise(resolve => {
-                pic.toBuffer().then(buffer => resolve(`data:image/jpeg;base64,${buffer.toString('base64')}`));
-            });
-            const promise = new Promise((resolve, reject) => {
-                const width = 720;
-                const zoomFactor = 2;
-                logger.info(`shooting ${width}*${height} webshot for ${url}`);
-                this.browser.newPage()
-                    .then(page => {
-                    const startTime = new Date().getTime();
-                    const getTimerTime = () => new Date().getTime() - startTime;
-                    const getTimeout = () => Math.max(500, webshotDelay - getTimerTime());
-                    page.setUserAgent('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36')
-                        .then(() => page.setViewport({
-                        width: width / zoomFactor,
-                        height: height / zoomFactor,
-                        isMobile: true,
-                        deviceScaleFactor: zoomFactor,
-                    }))
-                        .then(() => page.setBypassCSP(true))
-                        .then(() => page.goto(url, { waitUntil: 'load', timeout: getTimeout() }))
-                        // hide header, "more options" button, like and retweet count
-                        .then(() => page.addStyleTag({
-                        content: 'header{display:none!important}path[d=\'M20.207 7.043a1 1 0 0 0-1.414 0L12 13.836 5.207 7.043a1 1 0 0 0-1.414 1.414l7.5 7.5a.996.996 0 0 0 1.414 0l7.5-7.5a1 1 0 0 0 0-1.414z\'],div[role=\'button\']{display: none;}',
-                    }))
-                        // remove listeners
-                        .then(() => page.evaluate(() => {
-                        const poll = setInterval(() => {
-                            document.querySelectorAll('div[data-testid="placementTracking"]').forEach(container => {
-                                if (container) {
-                                    container.innerHTML = container.innerHTML;
-                                    clearInterval(poll);
-                                }
-                            });
-                        }, 250);
-                    }))
-                        .then(() => page.waitForSelector('article', { timeout: getTimeout() }))
-                        .catch((err) => {
-                        if (err.name !== 'TimeoutError')
-                            throw err;
-                        logger.warn(`navigation timed out at ${getTimerTime()} seconds`);
-                        return null;
-                    })
-                        .then(handle => {
-                        if (handle === null)
-                            throw new puppeteer.errors.TimeoutError();
-                    })
-                        .then(() => page.evaluate(() => {
-                        const cardImg = document.querySelector('div[data-testid^="card.layout"][data-testid$=".media"] img');
-                        if (typeof (cardImg === null || cardImg === void 0 ? void 0 : cardImg.getAttribute('src')) === 'string') {
-                            const match = cardImg === null || cardImg === void 0 ? void 0 : cardImg.getAttribute('src').match(/^(.*\/card_img\/(\d+)\/.+\?format=.*)&name=/);
-                            if (match) {
-                                // tslint:disable-next-line: variable-name
-                                const [media_url_https, id_str] = match.slice(1);
-                                return {
-                                    media_url: media_url_https.replace(/^https/, 'http'),
-                                    media_url_https,
-                                    url: '',
-                                    display_url: '',
-                                    expanded_url: '',
-                                    type: 'photo',
-                                    id: Number(id_str),
-                                    id_str,
-                                    sizes: undefined,
-                                };
-                            }
-                        }
-                    }))
-                        .then(cardImg => { if (cardImg)
-                        this.extendEntity(cardImg); })
-                        .then(() => page.addScriptTag({
-                        content: 'document.documentElement.scrollTop=0;',
-                    }))
-                        .then(() => util_1.promisify(setTimeout)(getTimeout()))
-                        .then(() => page.screenshot())
-                        .then(screenshot => {
-                        new pngjs_1.PNG({
-                            filterType: 4,
-                            deflateLevel: 0,
-                        }).on('parsed', function () {
-                            // remove comment area
-                            // tslint:disable-next-line: no-shadowed-variable
-                            const idx = (x, y) => (this.width * y + x) << 2;
-                            let boundary = null;
-                            let x = zoomFactor * 2;
-                            for (let y = 0; y < this.height; y++) {
-                                if (this.data[idx(x, y)] !== 255 &&
-                                    this.data[idx(x, y)] === this.data[idx(x + zoomFactor * 10, y)]) {
-                                    if (this.data[idx(x, y + 18 * zoomFactor)] !== 255) {
-                                        // footer kicks in
-                                        boundary = null;
-                                    }
-                                    else {
-                                        boundary = y;
-                                    }
-                                    break;
-                                }
-                            }
-                            if (boundary !== null) {
-                                logger.info(`found boundary at ${boundary}, cropping image`);
-                                this.data = this.data.slice(0, idx(this.width, boundary));
-                                this.height = boundary;
-                                boundary = null;
-                                x = Math.floor(16 * zoomFactor);
-                                let flag = false;
-                                let cnt = 0;
-                                for (let y = this.height - 1; y >= 0; y--) {
-                                    if ((this.data[idx(x, y)] === 255) === flag) {
-                                        cnt++;
-                                        flag = !flag;
-                                    }
-                                    else
-                                        continue;
-                                    // line above the "comment", "retweet", "like", "share" button row
-                                    if (cnt === 2) {
-                                        boundary = y + 1;
-                                    }
-                                    // if there are a "retweet" count and "like" count row, this will be the line above it
-                                    if (cnt === 4) {
-                                        const b = y + 1;
-                                        if (this.height - boundary - (boundary - b) <= 1) {
-                                            boundary = b;
-                                            //   }
-                                            // }
-                                            // // if "retweet" count and "like" count are two rows, this will be the line above the first
-                                            // if (cnt === 6) {
-                                            //   const c = y + 1;
-                                            //   if (this.height - boundary - 2 * (boundary - c) <= 2) {
-                                            //     boundary = c;
-                                            break;
-                                        }
-                                    }
-                                }
-                                if (boundary != null) {
-                                    logger.info(`found boundary at ${boundary}, trimming image`);
-                                    this.data = this.data.slice(0, idx(this.width, boundary));
-                                    this.height = boundary;
-                                }
-                                sharpToBase64(jpeg(this.pack())).then(base64 => {
-                                    logger.info(`finished webshot for ${url}`);
-                                    resolve({ base64, boundary });
-                                });
-                            }
-                            else if (height >= 8 * 1920) {
-                                logger.warn('too large, consider as a bug, returning');
-                                sharpToBase64(jpeg(this.pack())).then(base64 => {
-                                    resolve({ base64, boundary: 0 });
-                                });
-                            }
-                            else {
-                                logger.info('unable to find boundary, try shooting a larger image');
-                                resolve({ base64: '', boundary });
-                            }
-                        }).parse(screenshot);
-                    })
-                        .catch(err => {
-                        if (err.name !== 'TimeoutError')
-                            throw err;
-                        logger.error(`error shooting webshot for ${url}, could not load web page of tweet`);
-                        resolve({ base64: '', boundary: 0 });
-                    })
-                        .finally(() => page.close());
-                })
-                    .catch(reject);
-            });
-            return promise.then(data => {
-                if (data.boundary === null)
-                    return this.renderWebshot(url, height + 1920, webshotDelay);
-                else
-                    return data.base64;
-            }).catch(error => new Promise(resolve => this.reconnect(error, resolve))
-                .then(() => this.renderWebshot(url, height, webshotDelay)));
-        };
         this.fetchMedia = (url) => {
             const gif = (data) => {
                 const matchDims = url.match(/\/(\d+)x(\d+)\//);
@@ -284,116 +86,58 @@ class Webshot extends CallableInstance {
                 });
             }).then(typedData => `data:${typedData.mimetype};base64,${Buffer.from(typedData.data).toString('base64')}`);
         };
-        // tslint:disable-next-line: no-conditional-assignment
-        if (this.mode = mode) {
-            onready();
-        }
-        else {
-            this.connect(onready);
-        }
+        this.mode = mode;
+        onready();
     }
-    webshot(tweets, uploader, callback, webshotDelay) {
+    webshot(user, fleets, uploader, callback, webshotDelay) {
         let promise = new Promise(resolve => {
             resolve();
         });
-        tweets.forEach(twi => {
+        fleets.forEach(fleet => {
+            var _a, _b;
             promise = promise.then(() => {
-                logger.info(`working on ${twi.user.screen_name}/${twi.id_str}`);
+                logger.info(`working on ${user.screen_name}/${fleet.fleet_id}`);
             });
-            const originTwi = twi.retweeted_status || twi;
             const messageChain = [];
             // text processing
-            let author = `${twi.user.name} (@${twi.user.screen_name}):\n`;
-            if (twi.retweeted_status)
-                author += `RT @${twi.retweeted_status.user.screen_name}: `;
-            let text = originTwi.full_text;
-            promise = promise.then(() => {
-                if (originTwi.entities && originTwi.entities.urls && originTwi.entities.urls.length) {
-                    originTwi.entities.urls.forEach(url => {
-                        text = text.replace(new RegExp(url.url, 'gm'), url.expanded_url);
-                    });
-                }
-                if (originTwi.extended_entities) {
-                    originTwi.extended_entities.media.forEach(media => {
-                        text = text.replace(new RegExp(media.url, 'gm'), this.mode === 1 ? typeInZH[media.type] : '');
-                    });
-                }
-                if (this.mode > 0)
-                    messageChain.push(mirai_1.Message.Plain(author + xmlEntities.decode(text)));
-            });
-            // invoke webshot
-            if (this.mode === 0) {
-                const url = `https://mobile.twitter.com/${twi.user.screen_name}/status/${twi.id_str}`;
-                this.extendEntity = (cardImg) => {
-                    var _a, _b;
-                    originTwi.extended_entities = Object.assign(Object.assign({}, originTwi.extended_entities), { media: [
-                            ...(_b = (_a = originTwi.extended_entities) === null || _a === void 0 ? void 0 : _a.media) !== null && _b !== void 0 ? _b : [],
-                            cardImg,
-                        ] });
-                };
-                promise = promise.then(() => this.renderWebshot(url, 1920, webshotDelay))
-                    .then(base64url => {
-                    if (base64url)
-                        return uploader(mirai_1.Message.Image('', base64url, url), () => mirai_1.Message.Plain(author + text));
-                    return mirai_1.Message.Plain(author + text);
-                })
-                    .then(msg => {
-                    if (msg)
-                        messageChain.push(msg);
-                });
-            }
+            let author = `${user.name} (@${user.screen_name}):\n`;
+            let date = `${new Date(fleet.created_at)}\n`;
+            let text = (_b = author + date + ((_a = fleet.media_bounding_boxes) === null || _a === void 0 ? void 0 : _a.map(box => box.entity.value).join('\n'))) !== null && _b !== void 0 ? _b : '';
+            messageChain.push(mirai_1.Message.Plain(author + date));
             // fetch extra entities
             // tslint:disable-next-line: curly
             if (1 - this.mode % 2)
                 promise = promise.then(() => {
-                    if (originTwi.extended_entities) {
-                        return utils_1.chainPromises(originTwi.extended_entities.media.map(media => {
-                            let url;
-                            if (media.type === 'photo') {
-                                url = media.media_url_https.replace(/\.([a-z]+)$/, '?format=$1') + '&name=orig';
-                            }
-                            else {
-                                url = media.video_info.variants
-                                    .filter(variant => variant.bitrate !== undefined)
-                                    .sort((var1, var2) => var2.bitrate - var1.bitrate)
-                                    .map(variant => variant.url)[0]; // largest video
-                            }
-                            const altMessage = mirai_1.Message.Plain(`\n[失败的${typeInZH[media.type].type}：${url}]`);
-                            return this.fetchMedia(url)
-                                .then(base64url => uploader(mirai_1.Message.Image('', base64url, media.type === 'photo' ? url : `${url} as gif`), () => altMessage))
-                                .catch(error => {
-                                logger.warn('unable to fetch media, sending plain text instead...');
-                                return altMessage;
-                            })
-                                .then(msg => {
-                                messageChain.push(msg);
-                            });
-                        }));
+                    const media = fleet.media_entity;
+                    let url;
+                    if (fleet.media_key.media_category === 'TWEET_IMAGE') {
+                        media.type = 'photo';
+                        url = media.media_url_https.replace(/\.([a-z]+)$/, '?format=$1') + '&name=orig';
                     }
-                });
-            // append URLs, if any
-            if (this.mode === 0) {
-                if (originTwi.entities && originTwi.entities.urls && originTwi.entities.urls.length) {
-                    promise = promise.then(() => {
-                        const urls = originTwi.entities.urls
-                            .filter(urlObj => urlObj.indices[0] < originTwi.display_text_range[1])
-                            .map(urlObj => `\n\ud83d\udd17 ${urlObj.expanded_url}`);
-                        if (urls.length) {
-                            messageChain.push(mirai_1.Message.Plain(urls.join('')));
-                        }
+                    else {
+                        media.type = fleet.media_key.media_category === 'TWEET_VIDEO' ? 'video' : 'animated_gif';
+                        media.video_info = media.media_info.video_info;
+                        text += `[${typeInZH[media.type].type}]`;
+                        url = media.video_info.variants // bitrate -> bit_rate
+                            .filter(variant => variant.bit_rate !== undefined)
+                            .sort((var1, var2) => var2.bit_rate - var1.bit_rate)
+                            .map(variant => variant.url)[0]; // largest video
+                    }
+                    const altMessage = mirai_1.Message.Plain(`\n[失败的${typeInZH[media.type].type}：${url}]`);
+                    return this.fetchMedia(url)
+                        .then(base64url => uploader(mirai_1.Message.Image('', base64url, media.type === 'photo' ? url : `${url} as gif`), () => altMessage))
+                        .catch(error => {
+                        logger.warn('unable to fetch media, sending plain text instead...');
+                        return altMessage;
+                    })
+                        .then(msg => {
+                        messageChain.push(msg);
                     });
-                }
-            }
-            // refer to quoted tweet, if any
-            if (originTwi.is_quote_status) {
-                promise = promise.then(() => {
-                    messageChain.push(mirai_1.Message.Plain(`\n回复此命令查看引用的推文：\n/twitter_view ${originTwi.quoted_status_permalink.expanded}`));
                 });
-            }
             promise.then(() => {
-                logger.info(`done working on ${twi.user.screen_name}/${twi.id_str}, message chain:`);
+                logger.info(`done working on ${user.screen_name}/${fleet.fleet_id}, message chain:`);
                 logger.info(JSON.stringify(messageChain));
-                callback(messageChain, xmlEntities.decode(text), author);
+                callback(messageChain, xmlEntities.decode(text));
             });
         });
         return promise;

package.json

@@ -30,11 +30,12 @@
     "lint": "npx tslint --fix -c tslint.json --project ./"
   },
   "dependencies": {
+    "axios": "^0.21.1",
     "callable-instance": "^2.0.0",
     "command-line-usage": "^5.0.5",
     "html-entities": "^1.3.1",
     "log4js": "^6.3.0",
-    "mirai-ts": "github:CL-Jeremy/mirai-ts#built",
+    "mirai-ts": "^0.7.10",
     "pngjs": "^5.0.0",
     "puppeteer": "^2.1.0",
     "read-all-stream": "^3.1.0",
@@ -42,7 +43,7 @@
     "sharp": "^0.25.4",
     "temp": "^0.9.1",
     "twitter": "^1.7.1",
-    "typescript": "^4.0.2"
+    "typescript": "^4.1.3"
   },
   "devDependencies": {
     "@types/node": "^10.17.27",

src/command.ts

@@ -3,8 +3,7 @@ import * as path from 'path';
 
 import { relativeDate } from './datetime';
 import { getLogger } from './loggers';
-import { sendTimeline, sendTweet, ScreenNameNormalizer as normalizer } from './twitter';
-import { BigNumOps } from './utils';
+import { ScreenNameNormalizer as normalizer } from './twitter';
 
 const logger = getLogger('command');
 
@@ -33,13 +32,6 @@ function parseCmd(message: string): {
 
 function parseLink(link: string): string[] {
   let match =
-    link.match(/twitter.com\/([^\/?#]+)\/lists\/([^\/?#]+)/) ||
-    link.match(/^([^\/?#]+)\/([^\/?#]+)$/);
-  if (match) return [match[1], `/lists/${match[2]}`];
-  match =
-    link.match(/twitter.com\/([^\/?#]+)\/status\/(\d+)/);
-  if (match) return [match[1], `/status/${match[2]}`];
-  match =
     link.match(/twitter.com\/([^\/?#]+)/) ||
     link.match(/^([^\/?#]+)$/);
   if (match) return [match[1]];
@@ -71,36 +63,26 @@ function sub(chat: IChat, args: string[], reply: (msg: string) => any,
     return reply('请先添加机器人为好友。');
   }
   if (args.length === 0) {
-    return reply('找不到要订阅的链接。');
+    return reply('找不到要订阅推特故事的链接。');
   }
   const match = parseLink(args[0]);
   if (!match) {
     return reply(`订阅链接格式错误：
-示例：
-https://twitter.com/Saito_Shuka
-https://twitter.com/rikakomoe/lists/lovelive
-https://twitter.com/TomoyoKurosawa/status/1294613494860361729`);
-  }
-  let offset = '0';
-  if (match[1]) {
-    const matchStatus = match[1].match(/\/status\/(\d+)/);
-    if (matchStatus) {
-      offset = BigNumOps.plus(matchStatus[1], '-1');
-      delete match[1];
-    }
+示例：https://twitter.com/sunflower930316`);
   }
   const subscribeTo = (link: string, config: {addNew?: boolean, msg?: string} = {}) => {
-    const {addNew = false, msg = `已为此聊天订阅 ${link}`} = config;
+    const {addNew = false, msg = `已为此聊天订阅 ${link} 的推特故事`} = config;
     if (addNew) {
       lock.feed.push(link);
       lock.threads[link] = {
-        offset,
+        permaFeed: normalizer.permaFeeds[link],
+        offset: '0',
         subscribers: [],
         updatedAt: '',
       };
     }
     lock.threads[link].subscribers.push(chat);
-    logger.warn(`chat ${JSON.stringify(chat)} has subscribed ${link}`);
+    logger.warn(`chat ${JSON.stringify(chat)} has subscribed fleets for ${link}`);
     fs.writeFileSync(path.resolve(lockfile), JSON.stringify(lock));
     reply(msg);
   };
@@ -108,17 +90,9 @@ https://twitter.com/TomoyoKurosawa/status/1294613494860361729`);
   if (index > -1) return reply('此聊天已订阅此链接。');
   if (realLink) return subscribeTo(realLink);
   const [rawUserName, more] = match;
-  if (rawUserName.toLowerCase() === 'i' && more?.match(/lists\/(\d+)/)) {
-    return subscribeTo(linkBuilder('i', more), {addNew: true});
-  }
   normalizer.normalizeLive(rawUserName).then(userName => {
     if (!userName) return reply(`找不到用户 ${rawUserName.replace(/^@?(.*)$/, '@$1')}。`);
-    const link = linkBuilder(userName, more);
-    const msg = (offset === '0') ?
-      undefined :
-        `已为此聊天订阅 ${link} 并回溯到此动态 ID（含）之后的第一条动态。
-（参见：https://blog.twitter.com/engineering/en_us/a/2010/announcing-snowflake.html）`;
-    subscribeTo(link, {addNew: true, msg});
+    subscribeTo(linkBuilder(userName, more), {addNew: true});
   });
 }
 
@@ -129,19 +103,19 @@ function unsub(chat: IChat, args: string[], reply: (msg: string) => any,
     return reply('请先添加机器人为好友。');
   }
   if (args.length === 0) {
-    return reply('找不到要退订的链接。');
+    return reply('找不到要退订推特故事的链接。');
   }
   const match = parseLink(args[0]);
   if (!match) {
     return reply('链接格式有误。');
   }
   const [link, index] = linkFinder(match, chat, lock);
-  if (index === -1) return list(chat, args, msg => reply('您没有订阅此链接。\n' + msg), lock);
+  if (index === -1) return list(chat, args, msg => reply('您没有订阅此链接的推特故事。\n' + msg), lock);
   else {
     lock.threads[link].subscribers.splice(index, 1);
     fs.writeFileSync(path.resolve(lockfile), JSON.stringify(lock));
     logger.warn(`chat ${JSON.stringify(chat)} has unsubscribed ${link}`);
-    return reply(`已为此聊天退订 ${link}`);
+    return reply(`已为此聊天退订 ${link} 的推特故事`);
   }
 }
 
@@ -155,66 +129,7 @@ function list(chat: IChat, _: string[], reply: (msg: string) => any, lock: ILock
       chat.chatID === chatID && chat.chatType === chatType
     )) links.push(`${key} ${relativeDate(lock.threads[key].updatedAt)}`);
   });
-  return reply('此聊天中订阅的链接：\n' + links.join('\n'));
-}
-
-function view(chat: IChat, args: string[], reply: (msg: string) => any): void {
-  if (args.length === 0) {
-    return reply('找不到要查看的链接。');
-  }
-  const match = args[0].match(/^(?:.*twitter.com\/[^\/?#]+\/status\/)?(\d+)/);
-  if (!match) {
-    return reply('链接格式有误。');
-  }
-  try {
-    sendTweet(match[1], chat);
-  } catch (e) {
-    reply('推特机器人尚未加载完毕，请稍后重试。');
-  }
-}
-
-function query(chat: IChat, args: string[], reply: (msg: string) => any): void {
-  if (args.length === 0) {
-    return reply('找不到要查询的用户。');
-  }
-  const match = 
-    args[0].match(/twitter.com\/([^\/?#]+)/) ||
-    args[0].match(/^([^\/?#]+)$/);
-  if (!match) {
-    return reply('链接格式有误。');
-  }
-  const conf: {
-    username: string,
-    count?: string,
-    since?: string,
-    until?: string,
-    noreps: string,
-    norts: string,
-  } = {username: match[1], noreps: 'on', norts: 'off'};
-  const confZH: Record<Exclude<keyof typeof conf, 'username'>, string> = {
-    count: '数量上限',
-    since: '起始点',
-    until: '结束点',
-    noreps: '忽略回复推文（on/off）',
-    norts: '忽略原生转推（on/off）',
-  };
-  for (const arg of args.slice(1)) {
-    const optMatch = arg.match(/^(count|since|until|noreps|norts)=(.*)/);
-    if (!optMatch) return reply(`未定义的查询参数：${arg}。`);
-    const optKey = optMatch[1] as keyof typeof confZH;
-    if (optMatch.length === 1) return reply(`查询${confZH[optKey]}参数格式有误。`);
-    conf[optKey] = optMatch[2];
-    if (optMatch[2] === '') return reply(`查询${confZH[optKey]}参数值不可为空。`);
-  }
-  if (conf.count !== undefined && !Number(conf.count) || Math.abs(Number(conf.count)) > 50) {
-    return reply('查询数量上限参数为零、非数值或超出取值范围。');
-  }
-  try {
-    sendTimeline(conf, chat);
-  } catch (e) {
-    logger.error(`error querying timeline, error: ${e}`);
-    reply('推特机器人尚未加载完毕，请稍后重试。');
-  }
+  return reply('此聊天中订阅推特故事的链接：\n' + links.join('\n'));
 }
 
-export { parseCmd, sub, list, unsub, view, query };
+export { parseCmd, sub, list, unsub };

src/helper.ts

@@ -0,0 +1,24 @@
+interface ICommand {
+  cmd: string;
+  args: string[];
+}
+
+export default function (message: string): ICommand {
+  message = message.trim();
+  message = message.replace('\\\\', '\\0x5c');
+  message = message.replace('\\\"', '\\0x22');
+  message = message.replace('\\\'', '\\0x27');
+  const strs = message.match(/'[\s\S]*?'|"[\s\S]*?"|\S*\[CQ:[\s\S]*?\]\S*|\S+/mg);
+  const cmd = strs?.length ? strs[0].length ? strs[0].substring(0, 1) === '/' ? strs[0].substring(1) : '' : '' : '';
+  const args = strs?.slice(1).map(arg => {
+    arg = arg.replace(/^["']+|["']+$/g, '');
+    arg = arg.replace('\\0x27', '\\\'');
+    arg = arg.replace('\\0x22', '\\\"');
+    arg = arg.replace('\\0x5c', '\\\\');
+    return arg;
+  });
+  return {
+    cmd,
+    args,
+  };
+}

src/main.ts

@@ -145,6 +145,8 @@ const worker = new Worker({
   consumer_secret: config.twitter_consumer_secret,
   access_token_key: config.twitter_access_token_key,
   access_token_secret: config.twitter_access_token_secret,
+  private_auth_token: config.twitter_private_auth_token,
+  private_csrf_token: config.twitter_private_csrf_token,
   lock,
   lockfile: config.lockfile,
   workInterval: config.work_interval,

src/mirai.ts

@@ -4,7 +4,7 @@ import Mirai, { MessageType } from 'mirai-ts';
 import MiraiMessage from 'mirai-ts/dist/message';
 import * as temp from 'temp';
 
-import { parseCmd, query, view } from './command';
+import { parseCmd } from './command';
 import { getLogger } from './loggers';
 
 const logger = getLogger('qqbot');
@@ -137,7 +137,7 @@ export default class {
       port: this.botInfo.port,
     });
 
-    this.bot.axios.defaults.maxContentLength = Infinity;
+    this.bot.axios.defaults.maxContentLength = this.bot.axios.defaults.maxBodyLength = Infinity;
 
     this.bot.on('NewFriendRequestEvent', evt => {
       logger.debug(`detected new friend request event: ${JSON.stringify(evt)}`);
@@ -148,7 +148,7 @@ export default class {
         permission: 'OWNER' | 'ADMINISTRATOR' | 'MEMBER',
       }]) => {
         if (groupList.some(groupItem => groupItem.id === evt.groupId)) {
-          evt.respond('allow');
+          evt.respond(0); // allow
           return logger.info(`accepted friend request from ${evt.fromId} (from group ${evt.groupId})`);
         }
         logger.warn(`received friend request from ${evt.fromId} (from group ${evt.groupId})`);
@@ -165,7 +165,7 @@ export default class {
         remark: string,
       }]) => {
         if (friendList.some(friendItem => friendItem.id = evt.fromId)) {
-          evt.respond('allow');
+          evt.respond(0); // allow
           return logger.info(`accepted group invitation from ${evt.fromId} (friend)`);
         }
         logger.warn(`received group invitation from ${evt.fromId} (unknown)`);
@@ -177,58 +177,23 @@ export default class {
       const chat = await this.getChat(msg);
       const cmdObj = parseCmd(msg.plain);
       switch (cmdObj.cmd) {
-        case 'twitter_view':
-        case 'twitter_get':
-          view(chat, cmdObj.args, msg.reply);
-          break;
-        case 'twitter_query':
-        case 'twitter_gettimeline':
-          query(chat, cmdObj.args, msg.reply);
-          break;
-        case 'twitter_sub':
-        case 'twitter_subscribe':
+        case 'twitterfleets_sub':
+        case 'twitterfleets_subscribe':
           this.botInfo.sub(chat, cmdObj.args, msg.reply);
           break;
-        case 'twitter_unsub':
-        case 'twitter_unsubscribe':
+        case 'twitterfleets_unsub':
+        case 'twitterfleets_unsubscribe':
           this.botInfo.unsub(chat, cmdObj.args, msg.reply);
           break;
         case 'ping':
-        case 'twitter':
+        case 'twitterfleets':
           this.botInfo.list(chat, cmdObj.args, msg.reply);
           break;
         case 'help':
-          if (cmdObj.args.length === 0) {
-            msg.reply(`推特搬运机器人：
-/twitter - 查询当前聊天中的推文订阅
-/twitter_subscribe〈链接|用户名〉- 订阅 Twitter 推文搬运
-/twitter_unsubscribe〈链接|用户名〉- 退订 Twitter 推文搬运
-/twitter_view〈链接〉- 查看推文
-/twitter_query〈链接|用户名〉[参数列表...] - 查询时间线（详见 /help twitter_query）\
-${chat.chatType === ChatType.Temp ?
-  '\n（当前游客模式下无法使用订阅功能，请先添加本账号为好友。）' : ''
-}`);
-          } else if (cmdObj.args[0] === 'twitter_query') {
-            msg.reply(`查询时间线中的推文：
-/twitter_query〈链接|用户名〉[〈参数 1〉=〈值 1〉〈参数 2〉=〈值 2〉...]
-
-参数列表（方框内全部为可选，留空则为默认）：
-    count：查询数量上限（类型：非零整数，最大值正负 50）[默认值：10]
-    since：查询起始点（类型：正整数或日期）[默认值：（空，无限过去）]
-    until：查询结束点（类型：正整数或日期）[默认值：（空，当前时刻）]
-    noreps 忽略回复推文（类型：on/off）[默认值：on（是）]
-    norts：忽略原生转推（类型：on/off）[默认值：off（否）]`)
-              .then(() => msg.reply(`\
-起始点和结束点为正整数时取推特推文编号作为比较基准，否则会尝试作为日期读取。
-推荐的日期格式：2012-12-22 12:22 UTC+2 （日期和时间均为可选，可分别添加）
-count 为正时，从新向旧查询；为负时，从旧向新查询
-count 与 since/until 并用时，取二者中实际查询结果较少者
-例子：/twitter_query RiccaTachibana count=5 since="2019-12-30\
- UTC+9" until="2020-01-06 UTC+8" norts=on
-    从起始时间点（含）到结束时间点（不含）从新到旧获取最多 5 条推文，\
-其中不包含原生转推（实际上用户只发了 1 条）`)
-            );
-          }
+          msg.reply(`推特故事搬运机器人：
+/twitterfleets - 查询当前聊天中的推特故事订阅
+/twitterfleets_subscribe [链接] - 订阅 Twitter Fleets 搬运
+/twitterfleets_unsubscribe [链接] - 退订 Twitter Fleets 搬运`);
       }
     });
 }

src/model.d.ts

@@ -27,6 +27,7 @@ interface ILock {
   threads: {
     [key: string]:
       {
+        permaFeed: string,
         offset: string,
         updatedAt: string,
         subscribers: IChat[],

src/twitter.ts

@@ -1,11 +1,12 @@
 import * as fs from 'fs';
 import * as path from 'path';
+import * as request from 'request';
 import * as Twitter from 'twitter';
 import TwitterTypes from 'twitter-d';
 
 import { getLogger } from './loggers';
 import QQBot, { Message, MessageChain } from './mirai';
-import { chainPromises, BigNumOps } from './utils';
+import { BigNumOps } from './utils';
 import Webshot from './webshot';
 
 interface IWorkerOption {
@@ -18,6 +19,8 @@ interface IWorkerOption {
   consumer_secret: string;
   access_token_key: string;
   access_token_secret: string;
+  private_csrf_token: string;
+  private_auth_token: string;
   mode: number;
 }
 
@@ -26,6 +29,12 @@ export class ScreenNameNormalizer {
   // tslint:disable-next-line: variable-name
   public static _queryUser: (username: string) => Promise<string>;
 
+  public static permaFeeds = {};
+
+  public static savePermaFeedForUser(user: FullUser) {
+    this.permaFeeds[`https://twitter.com/${user.screen_name}`] = `https://twitter.com/i/user/${user.id_str}`;
+  }
+
   public static normalize = (username: string) => username.toLowerCase().replace(/^@/, '');
 
   public static async normalizeLive(username: string) {
@@ -43,31 +52,6 @@ export class ScreenNameNormalizer {
   }
 }
 
-export let sendTweet = (id: string, receiver: IChat): void => {
-  throw Error();
-};
-
-export interface ITimelineQueryConfig {
-  username: string;
-  count?: number;
-  since?: string;
-  until?: string;
-  noreps?: boolean;
-  norts?: boolean;
-}
-
-export let sendTimeline = (
-  conf: {[key in keyof ITimelineQueryConfig]: string},
-  receiver: IChat
-): void => {
-  throw Error();
-};
-
-const TWITTER_EPOCH = 1288834974657;
-const snowflake = (epoch: number) =>
-  Number.isNaN(epoch) ? undefined :
-    BigNumOps.lShift(String(epoch - 1 - TWITTER_EPOCH), 22);
-
 const logger = getLogger('twitter');
 const maxTrials = 3;
 const uploadTimeout = 10000;
@@ -99,21 +83,51 @@ const retryOnError = <T, U>(
 });
 
 export type FullUser = TwitterTypes.FullUser;
-export type Entities = TwitterTypes.Entities;
-export type ExtendedEntities = TwitterTypes.ExtendedEntities;
 export type MediaEntity = TwitterTypes.MediaEntity;
 
-interface ITweet extends TwitterTypes.Status {
-  user: FullUser;
-  retweeted_status?: Tweet;
+type TwitterMod = {
+  -readonly [K in keyof Twitter]: Twitter[K];
+} & {
+  options?: any;
 }
 
-export type Tweet = ITweet;
-export type Tweets = ITweet[];
+export type Fleet = {
+  created_at: string;
+  deleted_at: string;
+  expiration: string;
+  fleet_id: string;
+  fleet_thread_id: string;
+  media_bounding_boxes: [{
+    anchor_point_x: number;
+    anchor_point_y: number;
+    width: number;
+    height: number;
+    rotation: number;
+    entity: {
+        type: string;
+        value: any;
+    }
+  }];
+  media_entity: MediaEntity;
+  media_key: {
+    media_category: 'TWEET_IMAGE' | 'TWEET_VIDEO';
+    media_id: number;
+    media_id_str: string;
+  };
+  mentions: any;
+  mentions_str: any;
+  read: boolean;
+  text: string;
+  user_id: number;
+  user_id_str: string;
+};
+
+export type Fleets = Fleet[];
 
 export default class {
 
   private client: Twitter;
+  private privateClient: TwitterMod
   private lock: ILock;
   private lockfile: string;
   private workInterval: number;
@@ -129,53 +143,23 @@ export default class {
       access_token_key: opt.access_token_key,
       access_token_secret: opt.access_token_secret,
     });
+    this.privateClient = new Twitter({
+      bearer_token: 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
+    } as any);
+    this.privateClient.request = request.defaults({
+      headers: {
+        ...this.privateClient.options.request_options.headers,
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'Cookie': `auth_token=${opt.private_auth_token}; ct0=${opt.private_csrf_token};`,
+        'X-CSRF-Token': opt.private_csrf_token,
+      }
+    });
     this.lockfile = opt.lockfile;
     this.lock = opt.lock;
     this.workInterval = opt.workInterval;
     this.bot = opt.bot;
-    this.webshotDelay = opt.webshotDelay;
     this.mode = opt.mode;
     ScreenNameNormalizer._queryUser = this.queryUser;
-    sendTweet = (id, receiver) => {
-      this.getTweet(id, this.sendTweets(`tweet ${id}`, receiver))
-      .catch((err: {code: number, message: string}[]) => {
-        if (err[0].code !== 144) {
-          logger.warn(`error retrieving tweet: ${err[0].message}`);
-          this.bot.sendTo(receiver, `获取推文时出现错误：${err[0].message}`);
-        }
-        this.bot.sendTo(receiver, '找不到请求的推文，它可能已被删除。');
-      });
-    };
-    sendTimeline = ({username, count, since, until, noreps, norts}, receiver) => {
-      const countNum = Number(count) || 10;
-      (countNum > 0 ? this.queryTimeline : this.queryTimelineReverse)({
-        username,
-        count: Math.abs(countNum),
-        since: BigNumOps.parse(since) || snowflake(new Date(since).getTime()),
-        until: BigNumOps.parse(until) || snowflake(new Date(until).getTime()),
-        noreps: {on: true, off: false}[noreps],
-        norts: {on: true, off: false}[norts],
-      })
-      .then(tweets => chainPromises(
-        tweets.map(tweet => this.bot.sendTo(receiver, `\
-编号：${tweet.id_str}
-时间：${tweet.created_at}
-媒体：${tweet.extended_entities ? '有' : '无'}
-正文：\n${tweet.full_text.replace(/^([\s\S\n]{50})[\s\S\n]+?( https:\/\/t.co\/.*)?$/, '$1…$2')}`
-        ))
-        .concat(this.bot.sendTo(receiver, tweets.length ?
-          '时间线查询完毕，使用 /twitter_view <编号> 查看推文详细内容。' :
-            '时间线查询完毕，没有找到符合条件的推文。'
-        ))
-      ))
-      .catch((err: {code: number, message: string}[]) => {
-        if (err[0]?.code !== 34) {
-          logger.warn(`error retrieving timeline: ${err[0]?.message || err}`);
-          return this.bot.sendTo(receiver, `获取时间线时出现错误：${err[0]?.message || err}`);
-        }
-        this.bot.sendTo(receiver, `找不到用户 ${username.replace(/^@?(.*)$/, '@$1')}。`);
-      });
-    };
   }
 
   public launch = () => {
@@ -187,80 +171,15 @@ export default class {
 
   public queryUser = (username: string) =>
     this.client.get('users/show', {screen_name: username})
-    .then((user: FullUser) => user.screen_name)
-
-  public queryTimelineReverse = (conf: ITimelineQueryConfig) => {
-    if (!conf.since) return this.queryTimeline(conf);
-    const count = conf.count;
-    const maxID = conf.until;
-    conf.count = undefined;
-    const until = () =>
-      BigNumOps.min(maxID, BigNumOps.plus(conf.since, String(7 * 24 * 3600 * 1000 * 2 ** 22)));
-    conf.until = until();
-    const promise = (tweets: ITweet[]): Promise<ITweet[]> =>
-      this.queryTimeline(conf).then(newTweets => {
-        tweets = newTweets.concat(tweets);
-        conf.since = conf.until;
-        conf.until = until();
-        if (
-          tweets.length >= count ||
-          BigNumOps.compare(conf.since, conf.until) >= 0
-        ) {
-          return tweets.slice(-count);
-        }
-        return promise(tweets);
-      });
-    return promise([]);
-  }
-
-  public queryTimeline = (
-    { username, count, since, until, noreps, norts }: ITimelineQueryConfig
-  ) => {
-    username = username.replace(/^@?(.*)$/, '@$1');
-    logger.info(`querying timeline of ${username} with config: ${
-      JSON.stringify({
-        ...(count && {count}),
-        ...(since && {since}),
-        ...(until && {until}),
-        ...(noreps && {noreps}),
-        ...(norts && {norts}),
-    })}`);
-    const fetchTimeline = (
-      config = {
-        screen_name: username.slice(1),
-        trim_user: true,
-        exclude_replies: noreps ?? true,
-        include_rts: !(norts ?? false),
-        since_id: since,
-        max_id: until,
-        tweet_mode: 'extended',
-      },
-      tweets: ITweet[] = []
-    ): Promise<ITweet[]> =>
-      this.client.get('statuses/user_timeline', config)
-        .then((newTweets: ITweet[]) => {
-          if (newTweets.length) {
-            logger.debug(`fetched tweets: ${JSON.stringify(newTweets)}`);
-            config.max_id = BigNumOps.plus('-1', newTweets[newTweets.length - 1].id_str);
-            logger.info(`timeline query of ${username} yielded ${
-              newTweets.length
-            } new tweets, next query will start at offset ${config.max_id}`);
-            tweets.push(...newTweets);
-          }
-          if (!newTweets.length || tweets.length >= count) {
-            logger.info(`timeline query of ${username} finished successfully, ${
-              tweets.length
-            } tweets have been fetched`);
-            return tweets.slice(0, count);
-          }
-          return fetchTimeline(config, tweets);
-        });
-    return fetchTimeline();
-  }
+    .then((user: FullUser) => {
+      ScreenNameNormalizer.savePermaFeedForUser(user);
+      return user.screen_name;
+    })
 
-  private workOnTweets = (
-    tweets: Tweets,
-    sendTweets: (msg: MessageChain, text: string, author: string) => void
+  private workOnFleets = (
+    user: FullUser,
+    fleets: Fleets,
+    sendFleets: (msg: MessageChain, text: string) => void
   ) => {
     const uploader = (
       message: ReturnType<typeof Message.Image>,
@@ -279,24 +198,11 @@ export default class {
         }
       });
     };
-    return this.webshot(tweets, uploader, sendTweets, this.webshotDelay);
-  }
-
-  public getTweet = (id: string, sender: (msg: MessageChain, text: string, author: string) => void) => {
-    const endpoint = 'statuses/show';
-    const config = {
-      id,
-      tweet_mode: 'extended',
-    };
-    return this.client.get(endpoint, config)
-    .then((tweet: Tweet) => {
-      logger.debug(`api returned tweet ${JSON.stringify(tweet)} for query id=${id}`);
-      return this.workOnTweets([tweet], sender);
-    });
+    return this.webshot(user, fleets, uploader, sendFleets, this.webshotDelay);
   }
 
-  private sendTweets = (source?: string, ...to: IChat[]) =>
-  (msg: MessageChain, text: string, author: string) => {
+  private sendFleets = (source?: string, ...to: IChat[]) =>
+  (msg: MessageChain, text: string) => {
     to.forEach(subscriber => {
       logger.info(`pushing data${source ? ` of ${source}` : ''} to ${JSON.stringify(subscriber)}`);
       retryOnError(
@@ -307,7 +213,7 @@ export default class {
         } else {
           logger.warn(`${count - 1} consecutive failures while sending` +
             'message chain, trying plain text instead...');
-          terminate(this.bot.sendTo(subscriber, author + text));
+          terminate(this.bot.sendTo(subscriber, text));
         }
       });
     });
@@ -337,80 +243,59 @@ export default class {
     const currentFeed = lock.feed[lock.workon];
     logger.debug(`pulling feed ${currentFeed}`);
 
-    const promise = new Promise(resolve => {
-      let match = currentFeed.match(/https:\/\/twitter.com\/([^\/]+)\/lists\/([^\/]+)/);
-      let config: any;
-      let endpoint: string;
-      if (match) {
-        if (match[1] === 'i') {
-          config = {
-            list_id: match[2],
-            tweet_mode: 'extended',
-          };
-        } else {
-          config = {
-            owner_screen_name: match[1],
-            slug: match[2],
-            tweet_mode: 'extended',
-          };
-        }
-        endpoint = 'lists/statuses';
-      } else {
-        match = currentFeed.match(/https:\/\/twitter.com\/([^\/]+)/);
-        if (match) {
-          config = {
-            screen_name: match[1],
-            exclude_replies: false,
-            tweet_mode: 'extended',
-          };
-          endpoint = 'statuses/user_timeline';
-        }
-      }
-
-      if (endpoint) {
-        const offset = lock.threads[currentFeed].offset as unknown as number;
-        if (offset > 0) config.since_id = offset;
-        this.client.get(endpoint, config, (error, tweets, response) => {
-          if (error) {
-            if (error instanceof Array && error.length > 0 && error[0].code === 34) {
-              logger.warn(`error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
-              lock.threads[currentFeed].subscribers.forEach(subscriber => {
-                logger.info(`sending notfound message of ${currentFeed} to ${JSON.stringify(subscriber)}`);
-                this.bot.sendTo(subscriber, `链接 ${currentFeed} 指向的用户或列表不存在，请退订。`).catch();
-              });
-            } else {
-              logger.error(`unhandled error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
-            }
-            resolve();
-          } else resolve(tweets);
-        });
-      }
+    type FleetFeed = {fleet_threads: {fleets: Fleets}[]};
+    let user: FullUser;
+    let match = currentFeed.match(/https:\/\/twitter.com\/([^\/]+)/);
+    if (match) match = lock.threads[currentFeed].permaFeed.match(/https:\/\/twitter.com\/i\/user\/([^\/]+)/);
+    if (!match) {
+      logger.error(`cannot get endpoint for feed ${currentFeed}`);
+      return;
+    }
+    let endpoint = `https://api.twitter.com/fleets/v1/user_fleets?user_id=${match[1]}`;
+    const promise = new Promise<FleetFeed | void>((resolve, reject) => {
+      this.privateClient.get(endpoint, (error, fleetFeed: FleetFeed, _) => {
+        if (error) reject(error);
+        else resolve(fleetFeed);
+      });
     });
 
-    promise.then((tweets: Tweets) => {
-      logger.debug(`api returned ${JSON.stringify(tweets)} for feed ${currentFeed}`);
+    this.client.get('users/show', {user_id: match[1]})
+    .then((fullUser: FullUser) => { user = fullUser; return promise; })
+    .catch(error => {
+      logger.error(`unhandled error on fetching fleets for ${currentFeed}: ${JSON.stringify(error)}`);
+    })
+    .then((fleetFeed: FleetFeed) => {
+      logger.debug(`private api returned ${JSON.stringify(fleetFeed)} for feed ${currentFeed}`);
+      logger.debug(`api returned ${JSON.stringify(user)} for owner of feed ${currentFeed}`);
       const currentThread = lock.threads[currentFeed];
 
       const updateDate = () => currentThread.updatedAt = new Date().toString();
-      if (!tweets || tweets.length === 0) { updateDate(); return; }
+      if (!fleetFeed || fleetFeed.fleet_threads.length === 0) { updateDate(); return; }
 
-      const topOfFeed = tweets[0].id_str;
-      const updateOffset = () => currentThread.offset = topOfFeed;
+      let fleets = fleetFeed.fleet_threads[0].fleets;
+      const bottomOfFeed = fleets.slice(-1)[0].fleet_id.substring(3);
+      const updateOffset = () => currentThread.offset = bottomOfFeed;
 
       if (currentThread.offset === '-1') { updateOffset(); return; }
-      if (currentThread.offset === '0') tweets.splice(1);
+      if (currentThread.offset !== '0') {
+        const readCount = fleets.findIndex(fleet => {
+          return Number(BigNumOps.plus(fleet.fleet_id.substring(3), `-${currentThread.offset}`)) > 0;
+        });
+        if (readCount === -1) return;
+        fleets = fleets.slice(readCount);
+      }
 
-      return this.workOnTweets(tweets, this.sendTweets(`thread ${currentFeed}`, ...currentThread.subscribers))
+      return this.workOnFleets(user, fleets, this.sendFleets(`thread ${currentFeed}`, ...currentThread.subscribers))
       .then(updateDate).then(updateOffset);
     })
-      .then(() => {
-        lock.workon++;
-        let timeout = this.workInterval * 1000 / lock.feed.length;
-        if (timeout < 1000) timeout = 1000;
-        fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
-        setTimeout(() => {
-          this.work();
-        }, timeout);
-      });
+    .then(() => {
+      lock.workon++;
+      let timeout = this.workInterval * 1000 / lock.feed.length;
+      if (timeout < 1000) timeout = 1000;
+      fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
+      setTimeout(() => {
+        this.work();
+      }, timeout);
+    })
   }
 }

src/twitter_test.js

@@ -1,31 +0,0 @@
-import * as path from 'path';
-
-import Worker from './twitter'
-import Webshot from './webshot';
-
-const configPath = './config.json';
-
-let worker;
-try {
-  const config = require(path.resolve(configPath));
-  worker = new Worker(
-    Object.fromEntries(Object.entries(config).map(([k, v]) => [k.replace('twitter_', ''), v]))
-  );
-} catch (e) {
-  console.log('Failed to parse config file: ', configPath);
-  process.exit(1);
-}
-const webshot = new Webshot(worker.mode, () => {
-  worker.webshot = webshot;
-  worker.getTweet('1296935552848035840', (msg, text, author) => {
-    console.log(author + text);
-    console.log(JSON.stringify(msg));
-  }).catch(console.log);
-  worker.getTweet('1296935552848035841', (msg, text, author) => {
-    console.log(author + text);
-    console.log(JSON.stringify(msg));
-  }).catch(console.log);
-});
-worker.queryUser('tomoyokurosawa').then(console.log).catch(console.log);
-worker.queryUser('tomoyourosawa').then(console.log).catch(console.log);
-worker.queryUser('@tomoyokurosawa').then(console.log).catch(console.log);

src/webshot.ts

@@ -1,18 +1,11 @@
 import axios from 'axios';
 import * as CallableInstance from 'callable-instance';
 import { XmlEntities } from 'html-entities';
-import { PNG } from 'pngjs';
-import * as puppeteer from 'puppeteer';
-import { Browser } from 'puppeteer';
-import * as sharp from 'sharp';
-import { Readable } from 'stream';
-import { promisify } from 'util';
 
 import gifski from './gifski';
 import { getLogger } from './loggers';
 import { Message, MessageChain } from './mirai';
-import { MediaEntity, Tweets } from './twitter';
-import { chainPromises } from './utils';
+import { Fleets, FullUser } from './twitter';
 
 const xmlEntities = new XmlEntities();
 
@@ -31,216 +24,16 @@ const logger = getLogger('webshot');
 
 class Webshot
 extends CallableInstance<
-  [Tweets, (...args) => Promise<any>, (...args) => void, number],
+  [FullUser, Fleets, (...args) => Promise<any>, (...args) => void, number],
   Promise<void>
 > {
 
-  private browser: Browser;
   private mode: number;
 
   constructor(mode: number, onready?: () => any) {
     super('webshot');
-    // tslint:disable-next-line: no-conditional-assignment
-    if (this.mode = mode) {
-      onready();
-    } else {
-      this.connect(onready);
-    }
-  }
-
-  // use local Chromium
-  private connect = (onready) => puppeteer.connect({browserURL: 'http://127.0.0.1:9222'})
-  .then(browser => this.browser = browser)
-  .then(() => {
-    logger.info('launched puppeteer browser');
-    if (onready) return onready();
-  })
-  .catch(error => this.reconnect(error, onready))
-
-  private reconnect = (error, onready?) => {
-    logger.error(`connection error, reason: ${error}`);
-    logger.warn('trying to reconnect in 2.5s...');
-    return promisify(setTimeout)(2500)
-    .then(() => this.connect(onready));
-  }
-
-  private extendEntity = (media: MediaEntity) => {
-    logger.info('not working on a tweet');
-  }
-
-  private renderWebshot = (url: string, height: number, webshotDelay: number): Promise<string> => {
-    const jpeg = (data: Readable) => data.pipe(sharp()).jpeg({quality: 90, trellisQuantisation: true});
-    const sharpToBase64 = (pic: sharp.Sharp) => new Promise<string>(resolve => {
-      pic.toBuffer().then(buffer => resolve(`data:image/jpeg;base64,${buffer.toString('base64')}`));
-    });
-    const promise = new Promise<{ base64: string, boundary: null | number }>((resolve, reject) => {
-      const width = 720;
-      const zoomFactor = 2;
-      logger.info(`shooting ${width}*${height} webshot for ${url}`);
-      this.browser.newPage()
-        .then(page => {
-          const startTime = new Date().getTime();
-          const getTimerTime = () => new Date().getTime() - startTime;
-          const getTimeout = () => Math.max(500, webshotDelay - getTimerTime());
-          page.setUserAgent('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36')
-            .then(() => page.setViewport({
-              width: width / zoomFactor,
-              height: height / zoomFactor,
-              isMobile: true,
-              deviceScaleFactor: zoomFactor,
-            }))
-            .then(() => page.setBypassCSP(true))
-            .then(() => page.goto(url, {waitUntil: 'load', timeout: getTimeout()}))
-            // hide header, "more options" button, like and retweet count
-            .then(() => page.addStyleTag({
-              content: 'header{display:none!important}path[d=\'M20.207 7.043a1 1 0 0 0-1.414 0L12 13.836 5.207 7.043a1 1 0 0 0-1.414 1.414l7.5 7.5a.996.996 0 0 0 1.414 0l7.5-7.5a1 1 0 0 0 0-1.414z\'],div[role=\'button\']{display: none;}',
-            }))
-            // remove listeners
-            .then(() => page.evaluate(() => {
-              const poll = setInterval(() => {
-                document.querySelectorAll('div[data-testid="placementTracking"]').forEach(container => {
-                  if (container) {
-                    container.innerHTML = container.innerHTML;
-                    clearInterval(poll);
-                  }
-                });
-              }, 250);
-            }))
-            .then(() => page.waitForSelector('article', {timeout: getTimeout()}))
-            .catch((err: Error): Promise<puppeteer.ElementHandle<Element> | null> => {
-              if (err.name !== 'TimeoutError') throw err;
-              logger.warn(`navigation timed out at ${getTimerTime()} seconds`);
-              return null;
-            })
-            .then(handle => {
-              if (handle === null) throw new puppeteer.errors.TimeoutError();
-            })
-            .then(() => page.evaluate(() => {
-              const cardImg = document.querySelector('div[data-testid^="card.layout"][data-testid$=".media"] img');
-              if (typeof cardImg?.getAttribute('src') === 'string') {
-                const match = cardImg?.getAttribute('src')
-                  .match(/^(.*\/card_img\/(\d+)\/.+\?format=.*)&name=/);
-                if (match) {
-                  // tslint:disable-next-line: variable-name
-                  const [media_url_https, id_str] = match.slice(1);
-                  return {
-                    media_url: media_url_https.replace(/^https/, 'http'),
-                    media_url_https,
-                    url: '',
-                    display_url: '',
-                    expanded_url: '',
-                    type: 'photo',
-                    id: Number(id_str),
-                    id_str,
-                    sizes: undefined,
-                  };
-                }
-              }
-            }))
-            .then(cardImg => { if (cardImg) this.extendEntity(cardImg); })
-            .then(() => page.addScriptTag({
-              content: 'document.documentElement.scrollTop=0;',
-            }))
-            .then(() => promisify(setTimeout)(getTimeout()))
-            .then(() => page.screenshot())
-            .then(screenshot => {
-              new PNG({
-                filterType: 4,
-                deflateLevel: 0,
-              }).on('parsed', function () {
-                // remove comment area
-                // tslint:disable-next-line: no-shadowed-variable
-                const idx = (x: number, y: number) => (this.width * y + x) << 2;
-                let boundary = null;
-                let x = zoomFactor * 2;
-                for (let y = 0; y < this.height; y++) {
-                  if (
-                    this.data[idx(x, y)] !== 255 &&
-                    this.data[idx(x, y)] === this.data[idx(x + zoomFactor * 10, y)]
-                  ) {
-                    if (this.data[idx(x, y + 18 * zoomFactor)] !== 255) {
-                      // footer kicks in
-                      boundary = null;
-                    } else {
-                      boundary = y;
-                    }
-                    break;
-                  }
-                }
-                if (boundary !== null) {
-                  logger.info(`found boundary at ${boundary}, cropping image`);
-                  this.data = this.data.slice(0, idx(this.width, boundary));
-                  this.height = boundary;
-
-                  boundary = null;
-                  x = Math.floor(16 * zoomFactor);
-                  let flag = false;
-                  let cnt = 0;
-                  for (let y = this.height - 1; y >= 0; y--) {
-                    if ((this.data[idx(x, y)] === 255) === flag) {
-                      cnt++;
-                      flag = !flag;
-                    } else continue;
-
-                    // line above the "comment", "retweet", "like", "share" button row
-                    if (cnt === 2) {
-                      boundary = y + 1;
-                    }
-
-                    // if there are a "retweet" count and "like" count row, this will be the line above it
-                    if (cnt === 4) {
-                      const b = y + 1;
-                      if (this.height - boundary - (boundary - b) <= 1) {
-                        boundary = b;
-                    //   }
-                    // }
-
-                    // // if "retweet" count and "like" count are two rows, this will be the line above the first
-                    // if (cnt === 6) {
-                    //   const c = y + 1;
-                    //   if (this.height - boundary - 2 * (boundary - c) <= 2) {
-                    //     boundary = c;
-                        break;
-                      }
-                    }
-                  }
-                  if (boundary != null) {
-                    logger.info(`found boundary at ${boundary}, trimming image`);
-                    this.data = this.data.slice(0, idx(this.width, boundary));
-                    this.height = boundary;
-                  }
-
-                  sharpToBase64(jpeg(this.pack())).then(base64 => {
-                    logger.info(`finished webshot for ${url}`);
-                    resolve({base64, boundary});
-                  });
-                } else if (height >= 8 * 1920) {
-                  logger.warn('too large, consider as a bug, returning');
-                  sharpToBase64(jpeg(this.pack())).then(base64 => {
-                    resolve({base64, boundary: 0});
-                  });
-                } else {
-                  logger.info('unable to find boundary, try shooting a larger image');
-                  resolve({base64: '', boundary});
-                }
-              }).parse(screenshot);
-            })
-            .catch(err => {
-              if (err.name !== 'TimeoutError') throw err;
-              logger.error(`error shooting webshot for ${url}, could not load web page of tweet`);
-              resolve({base64: '', boundary: 0});
-            })
-            .finally(() => page.close());
-        })
-        .catch(reject);
-    });
-    return promise.then(data => {
-      if (data.boundary === null) return this.renderWebshot(url, height + 1920, webshotDelay);
-      else return data.base64;
-    }).catch(error =>
-      new Promise(resolve => this.reconnect(error, resolve))
-      .then(() => this.renderWebshot(url, height, webshotDelay))
-    );
+    this.mode = mode;
+    onready();
   }
 
   private fetchMedia = (url: string): Promise<string> => {
@@ -299,119 +92,64 @@ extends CallableInstance<
   }
 
   public webshot(
-    tweets: Tweets,
+    user: FullUser,
+    fleets: Fleets,
     uploader: (
       img: ReturnType<typeof Message.Image>,
       lastResort: (...args) => ReturnType<typeof Message.Plain>)
       => Promise<ReturnType<typeof Message.Image | typeof Message.Plain>>,
-    callback: (msgs: MessageChain, text: string, author: string) => void,
+    callback: (msgs: MessageChain, text: string) => void,
     webshotDelay: number
   ): Promise<void> {
     let promise = new Promise<void>(resolve => {
       resolve();
     });
-    tweets.forEach(twi => {
+    fleets.forEach(fleet => {
       promise = promise.then(() => {
-        logger.info(`working on ${twi.user.screen_name}/${twi.id_str}`);
+        logger.info(`working on ${user.screen_name}/${fleet.fleet_id}`);
       });
-      const originTwi = twi.retweeted_status || twi;
       const messageChain: MessageChain = [];
 
       // text processing
-      let author = `${twi.user.name} (@${twi.user.screen_name}):\n`;
-      if (twi.retweeted_status) author += `RT @${twi.retweeted_status.user.screen_name}: `;
-
-      let text = originTwi.full_text;
-
-      promise = promise.then(() => {
-        if (originTwi.entities && originTwi.entities.urls && originTwi.entities.urls.length) {
-          originTwi.entities.urls.forEach(url => {
-            text = text.replace(new RegExp(url.url, 'gm'), url.expanded_url);
-          });
-        }
-        if (originTwi.extended_entities) {
-          originTwi.extended_entities.media.forEach(media => {
-            text = text.replace(new RegExp(media.url, 'gm'), this.mode === 1 ? typeInZH[media.type] : '');
-          });
-        }
-        if (this.mode > 0) messageChain.push(Message.Plain(author + xmlEntities.decode(text)));
-      });
+      let author = `${user.name} (@${user.screen_name}):\n`;
+      let date = `${new Date(fleet.created_at)}\n`;
+      let text = author + date + fleet.media_bounding_boxes?.map(box => box.entity.value).join('\n') ?? '';
+      messageChain.push(Message.Plain(author + date));
 
-      // invoke webshot
-      if (this.mode === 0) {
-        const url = `https://mobile.twitter.com/${twi.user.screen_name}/status/${twi.id_str}`;
-        this.extendEntity = (cardImg: MediaEntity) => {
-          originTwi.extended_entities = {
-            ...originTwi.extended_entities,
-            media: [
-              ...originTwi.extended_entities?.media ?? [],
-              cardImg,
-            ],
-          };
-        };
-        promise = promise.then(() => this.renderWebshot(url, 1920, webshotDelay))
-          .then(base64url => {
-            if (base64url) return uploader(Message.Image('', base64url, url), () => Message.Plain(author + text));
-            return Message.Plain(author + text);
-          })
-          .then(msg => {
-            if (msg) messageChain.push(msg);
-          });
-      }
       // fetch extra entities
       // tslint:disable-next-line: curly
       if (1 - this.mode % 2) promise = promise.then(() => {
-        if (originTwi.extended_entities) {
-          return chainPromises(originTwi.extended_entities.media.map(media => {
-            let url: string;
-            if (media.type === 'photo') {
-              url = media.media_url_https.replace(/\.([a-z]+)$/, '?format=$1') + '&name=orig';
-            } else {
-              url = media.video_info.variants
-                .filter(variant => variant.bitrate !== undefined)
-                .sort((var1, var2) => var2.bitrate - var1.bitrate)
-                .map(variant => variant.url)[0]; // largest video
-            }
-            const altMessage = Message.Plain(`\n[失败的${typeInZH[media.type].type}：${url}]`);
-            return this.fetchMedia(url)
-              .then(base64url =>
-                uploader(Message.Image('', base64url, media.type === 'photo' ? url : `${url} as gif`), () => altMessage)
-              )
-              .catch(error => {
-                logger.warn('unable to fetch media, sending plain text instead...');
-                return altMessage;
-              })
-              .then(msg => {
-                messageChain.push(msg);
-              });
-          }));
-        }
+          const media = fleet.media_entity;
+          let url: string;
+          if (fleet.media_key.media_category === 'TWEET_IMAGE') {
+            media.type = 'photo';
+            url = media.media_url_https.replace(/\.([a-z]+)$/, '?format=$1') + '&name=orig';
+          } else {
+            media.type = fleet.media_key.media_category === 'TWEET_VIDEO' ? 'video' : 'animated_gif';
+            media.video_info = (media as any).media_info.video_info;
+            text += `[${typeInZH[media.type].type}]`;
+            url = (media.video_info.variants as any) // bitrate -> bit_rate
+              .filter(variant => variant.bit_rate !== undefined)
+              .sort((var1, var2) => var2.bit_rate - var1.bit_rate)
+              .map(variant => variant.url)[0]; // largest video
+          }
+          const altMessage = Message.Plain(`\n[失败的${typeInZH[media.type].type}：${url}]`);
+          return this.fetchMedia(url)
+            .then(base64url =>
+              uploader(Message.Image('', base64url, media.type === 'photo' ? url : `${url} as gif`), () => altMessage)
+            )
+            .catch(error => {
+              logger.warn('unable to fetch media, sending plain text instead...');
+              return altMessage;
+            })
+            .then(msg => {
+              messageChain.push(msg);
+            });
       });
-      // append URLs, if any
-      if (this.mode === 0) {
-        if (originTwi.entities && originTwi.entities.urls && originTwi.entities.urls.length) {
-          promise = promise.then(() => {
-            const urls = originTwi.entities.urls
-              .filter(urlObj => urlObj.indices[0] < originTwi.display_text_range[1])
-              .map(urlObj => `\n\ud83d\udd17 ${urlObj.expanded_url}`);
-            if (urls.length) {
-              messageChain.push(Message.Plain(urls.join('')));
-            }
-          });
-        }
-      }
-      // refer to quoted tweet, if any
-      if (originTwi.is_quote_status) {
-        promise = promise.then(() => {
-          messageChain.push(
-            Message.Plain(`\n回复此命令查看引用的推文：\n/twitter_view ${originTwi.quoted_status_permalink.expanded}`)
-          );
-        });
-      }
       promise.then(() => {
-        logger.info(`done working on ${twi.user.screen_name}/${twi.id_str}, message chain:`);
+        logger.info(`done working on ${user.screen_name}/${fleet.fleet_id}, message chain:`);
         logger.info(JSON.stringify(messageChain));
-        callback(messageChain, xmlEntities.decode(text), author);
+        callback(messageChain, xmlEntities.decode(text));
       });
     });
     return promise;