init idoly

config.example.json

@@ -7,5 +7,8 @@
   "twitter_consumer_secret": "",
   "twitter_access_token_key": "",
   "twitter_access_token_secret": "",
-  "loglevel": "info"
+  "bilibili_cookie_sessdata": "",
+  "loglevel": "info",
+  "lockfile": "subscribers.lock",
+  "work_interval": 60
 }

dist/command.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.view = exports.parseCmd = void 0;
-const koishi_1 = require("./koishi");
+exports.status = exports.unsub = exports.sub = exports.parseCmd = void 0;
+const datetime_1 = require("./datetime");
 const twitter_1 = require("./twitter");
 function parseCmd(message) {
     message = message.trim();
@@ -23,23 +23,33 @@ function parseCmd(message) {
     };
 }
 exports.parseCmd = parseCmd;
-function view(_chat, args, reply) {
-    let query = Number(args[0]);
-    if (args[0] && !Number.isInteger(query))
-        return reply('查询格式有误。\n格式：/nanatsuu_view [〈整数〉]');
-    twitter_1.queryByRegExp('t7s_staff', /の『それゆけ！ナナスタ☆通信』第(\d+)話はこちら！/, 3600 * 24 * 7)
-        .then(match => {
-        if (!match)
-            throw Error();
-        if (!args[0])
-            query = Number(match[1]);
-        if (query < 0)
-            query += Number(match[1]);
-        if (query < 0 || query > Number(match[1])) {
-            return reply(`查询取值范围有误。当前可用的取值范围：${-Number(match[1])}～${Number(match[1])}`);
-        }
-        reply(`第 ${query} 话：\n` +
-            koishi_1.Message.Image(`https://d2n19nac4w0gh6.cloudfront.net/resource/images/webview/comic/story/comic_${String(query).padStart(3, '0')}.jpg`));
-    }).catch((err) => reply(`查询失败，请稍后重试。${err.message ? `原因：${err}` : ''}`));
+function sub(chat, args, reply, lock, lockfile) {
+    if (chat.chatType === "temp") {
+        return reply('请先添加机器人为好友。');
+    }
+    const index = lock.subscribers.findIndex(({ chatID, chatType }) => chat.chatID === chatID && chat.chatType === chatType);
+    if (index > -1)
+        return reply('此聊天已订阅 IDOLY PRIDE BWIKI 更新提醒。');
+    lock.subscribers.push(chat);
+    reply('已为此聊天订阅 IDOLY PRIDE BWIKI 更新提醒。');
 }
-exports.view = view;
+exports.sub = sub;
+function unsub(chat, args, reply, lock, lockfile) {
+    if (chat.chatType === "temp") {
+        return reply('请先添加机器人为好友。');
+    }
+    const index = lock.subscribers.findIndex(({ chatID, chatType }) => chat.chatID === chatID && chat.chatType === chatType);
+    if (index === -1)
+        return reply('此聊天未订阅 IDOLY PRIDE BWIKI 更新提醒。');
+    lock.subscribers.splice(index, 1);
+    reply('已为此聊天退订 IDOLY PRIDE BWIKI 更新提醒。');
+}
+exports.unsub = unsub;
+function status(chat, _, reply, lock) {
+    const lastAction = lock.lastActions.sort((a1, a2) => Date.parse(a2.timestamp) - Date.parse(a1.timestamp))[0];
+    reply(`IDOLY PRIDE 官方推特追踪情况：
+上次更新时间：${(0, datetime_1.relativeDate)(lastAction.timestamp || '')}
+上次更新内容：${(0, twitter_1.parseAction)(lastAction)}
+上次检查时间：${(0, datetime_1.relativeDate)(lock.updatedAt || '')}`);
+}
+exports.status = status;

dist/datetime.js

@@ -0,0 +1,51 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.relativeDate = void 0;
+function relativeDate(dtstr) {
+    if (!dtstr)
+        return '暂无数据';
+    const dt = new Date(dtstr);
+    const dateTimeStamp = dt.getTime();
+    const second = 1000;
+    const minute = second * 60;
+    const hour = minute * 60;
+    const day = hour * 24;
+    const month = day * 30;
+    const now = new Date().getTime();
+    const diffValue = now - dateTimeStamp;
+    if (diffValue < 0) {
+        return;
+    }
+    const monthC = diffValue / month;
+    const weekC = diffValue / (7 * day);
+    const dayC = diffValue / day;
+    const hourC = diffValue / hour;
+    const minC = diffValue / minute;
+    const secC = diffValue / second;
+    let result;
+    if (monthC > 12) {
+        const y = dt.getFullYear() + ' 年';
+        const m = dt.getMonth() + 1 + ' 月';
+        const d = dt.getDate() + ' 日';
+        result = [y, m, d].join(' ');
+    }
+    else if (monthC >= 1) {
+        result = '' + Math.floor(monthC) + ' 个月前';
+    }
+    else if (weekC >= 1) {
+        result = '' + Math.floor(weekC) + ' 周前';
+    }
+    else if (dayC >= 1) {
+        result = '' + Math.floor(dayC) + ' 天前';
+    }
+    else if (hourC >= 1) {
+        result = '' + Math.floor(hourC) + ' 小时前';
+    }
+    else if (minC >= 1) {
+        result = '' + Math.floor(minC) + ' 分钟前';
+    }
+    else
+        result = '' + Math.floor(secC) + ' 秒前';
+    return result;
+}
+exports.relativeDate = relativeDate;

dist/koishi.js

@@ -15,7 +15,7 @@ require("koishi-adapter-onebot");
 const command_1 = require("./command");
 const loggers_1 = require("./loggers");
 const utils_1 = require("./utils");
-const logger = loggers_1.getLogger('qqbot');
+const logger = (0, loggers_1.getLogger)('qqbot');
 const cqUrlFix = (factory) => (...args) => factory(...args).replace(/(?<=\[CQ:.*)url=(?=(base64|file|https?):\/\/)/, 'file=');
 exports.Message = {
     Image: cqUrlFix(koishi_1.segment.image),
@@ -51,7 +51,7 @@ class default_1 {
             var _a, _b;
             let wasEmpty = false;
             const queue = (_a = this.messageQueues)[_b = `${type}:${id}`] || (_a[_b] = (() => { wasEmpty = true; return []; })());
-            queue.push(() => koishi_1.sleep(200).then(resolver));
+            queue.push(() => (0, koishi_1.sleep)(200).then(resolver));
             logger.debug(`no. of message currently queued for ${type}:${id}: ${queue.length}`);
             if (wasEmpty)
                 this.next(type, id);
@@ -139,7 +139,7 @@ class default_1 {
                             .then(() => { logger.info(`accepted friend request from ${userString} (from group ${groupString})`); })
                             .catch(error => { logger.error(`error accepting friend request from ${userString}, error: ${error}`); });
                     }
-                    utils_1.chainPromises(groupList.map(groupItem => (done) => Promise.resolve(done ||
+                    (0, utils_1.chainPromises)(groupList.map(groupItem => (done) => Promise.resolve(done ||
                         this.bot.getGroupMember(groupItem.groupId, session.userId).then(() => {
                             groupString = `${groupItem.groupName}(${groupItem.groupId})`;
                             return session.bot.handleFriendRequest(session.messageId, true)
@@ -170,19 +170,30 @@ class default_1 {
             }));
             this.app.middleware((session) => __awaiter(this, void 0, void 0, function* () {
                 const chat = yield this.getChat(session);
-                const cmdObj = command_1.parseCmd(session.content);
+                const cmdObj = (0, command_1.parseCmd)(session.content);
                 const reply = (msg) => __awaiter(this, void 0, void 0, function* () {
                     const userString = `${session.username}(${session.userId})`;
                     return (chat.chatType === "group" ? this.sendToGroup : this.sendToUser)(chat.chatID.toString(), msg)
                         .catch(error => { logger.error(`error replying to message from ${userString}, error: ${error}`); });
                 });
                 switch (cmdObj.cmd) {
-                    case 'nanatsuu_view':
-                        command_1.view(chat, cmdObj.args, reply);
+                    case 'ipwikistatus_sub':
+                        this.botInfo.sub(chat, cmdObj.args, reply);
+                        break;
+                    case 'ipwikistatus_unsub':
+                        this.botInfo.unsub(chat, cmdObj.args, reply);
+                        break;
+                    case 'ipwikistatus':
+                        this.botInfo.status(chat, cmdObj.args, reply);
                         break;
                     case 'help':
-                        if (cmdObj.args.length === 0) {
-                            reply('Nanasta 通信搬运机器人：\n/nanatsuu_view [〈整数〉] - 查看最新或指定的 Nanasta 通信话数');
+                        if (cmdObj.args[1] === 'ipwikistatus') {
+                            reply(`IDOLY PRIDE BWIKI 机器人：
+/ipwikistatus - 查询当前状态
+/ipwikistatus_sub - 订阅状态更新
+/ipwikistatus_unsub - 退订状态更新\
+${chat.chatType === "temp" ?
+                                '\n（当前游客模式下无法使用订阅功能，请先添加本账号为好友。）' : ''}`);
                         }
                 }
             }), true);
@@ -194,7 +205,7 @@ class default_1 {
             }
             catch (err) {
                 logger.error(`error connecting to bot provider at ${this.app.options.server}, will retry in 2.5s...`);
-                yield koishi_1.sleep(2500);
+                yield (0, koishi_1.sleep)(2500);
                 yield this.listen('retry connecting...');
             }
         });

dist/loggers.js

@@ -4,7 +4,7 @@ exports.setLogLevels = exports.getLogger = void 0;
 const log4js_1 = require("log4js");
 const loggers = [];
 function getLogger(category) {
-    const l = log4js_1.getLogger(category);
+    const l = (0, log4js_1.getLogger)(category);
     l.level = 'info';
     loggers.push(l);
     return l;

dist/main.js

@@ -8,17 +8,18 @@ const exampleConfig = require("../config.example.json");
 const loggers_1 = require("./loggers");
 const koishi_1 = require("./koishi");
 const twitter_1 = require("./twitter");
-const logger = loggers_1.getLogger();
+const command_1 = require("./command");
+const logger = (0, loggers_1.getLogger)();
 const sections = [
     {
-        header: 'GoCQHTTP Nana Bot',
-        content: 'The QQ Bot that does stuff.',
+        header: 'GoCQHTTP IPWIKI Bot',
+        content: 'The QQ Bot that updates Idoly Pride BWIKI.',
     },
     {
         header: 'Synopsis',
         content: [
-            '$ cq-nana-bot {underline config.json}',
-            '$ cq-nana-bot {bold --help}',
+            '$ cq-ipwiki-bot {underline config.json}',
+            '$ cq-ipwiki-bot {bold --help}',
         ],
     },
     {
@@ -47,13 +48,13 @@ catch (e) {
 }
 const requiredFields = [
     'twitter_consumer_key', 'twitter_consumer_secret', 'twitter_access_token_key', 'twitter_access_token_secret',
-    'cq_bot_qq',
+    'cq_bot_qq', 'bilibili_cookie_sessdata'
 ];
 const warningFields = [
     'cq_ws_host', 'cq_ws_port', 'cq_access_token',
 ];
 const optionalFields = [
-    'loglevel',
+    'lockfile', 'work_interval', 'loglevel',
 ].concat(warningFields);
 if (requiredFields.some((value) => config[value] === undefined)) {
     console.log(`${requiredFields.join(', ')} are required`);
@@ -66,17 +67,62 @@ optionalFields.forEach(key => {
         config[key] = exampleConfig[key];
     }
 });
-loggers_1.setLogLevels(config.loglevel);
+(0, loggers_1.setLogLevels)(config.loglevel);
+let lock;
+if (fs.existsSync(path.resolve(config.lockfile))) {
+    try {
+        lock = JSON.parse(fs.readFileSync(path.resolve(config.lockfile), 'utf8'));
+    }
+    catch (err) {
+        logger.error(`Failed to parse lockfile ${config.lockfile}: `, err);
+        lock = {
+            offset: '0',
+            lastActions: [],
+            subscribers: [],
+            updatedAt: undefined,
+        };
+    }
+    fs.access(path.resolve(config.lockfile), fs.constants.W_OK, err => {
+        if (err) {
+            logger.fatal(`cannot write lockfile ${path.resolve(config.lockfile)}, permission denied`);
+            process.exit(1);
+        }
+    });
+}
+else {
+    lock = {
+        offset: '0',
+        lastActions: [],
+        subscribers: [],
+        updatedAt: undefined,
+    };
+    try {
+        fs.writeFileSync(path.resolve(config.lockfile), JSON.stringify(lock));
+    }
+    catch (err) {
+        logger.fatal(`cannot write lockfile ${path.resolve(config.lockfile)}, permission denied`);
+        process.exit(1);
+    }
+}
 const qq = new koishi_1.default({
     access_token: config.cq_access_token,
     host: config.cq_ws_host,
     port: config.cq_ws_port,
     bot_id: config.cq_bot_qq,
+    status: (c, a, cb) => (0, command_1.status)(c, a, cb, lock),
+    sub: (c, a, cb) => (0, command_1.sub)(c, a, cb, lock, config.lockfile),
+    unsub: (c, a, cb) => (0, command_1.unsub)(c, a, cb, lock, config.lockfile),
 });
-new twitter_1.default({
+const worker = new twitter_1.default({
     consumerKey: config.twitter_consumer_key,
     consumerSecret: config.twitter_consumer_secret,
     accessTokenKey: config.twitter_access_token_key,
     accessTokenSecret: config.twitter_access_token_secret,
+    wikiSessionCookie: config.bilibili_cookie_sessdata,
+    lock,
+    lockfile: config.lockfile,
+    workInterval: config.work_interval,
+    bot: qq,
 });
+worker.launch();
 qq.connect();

dist/twitter.js

@@ -1,67 +1,165 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.queryByRegExp = exports.snowflake = void 0;
+exports.snowflake = exports.parseAction = exports.processTweetBody = exports.recurringKeywords = exports.keywordMap = void 0;
+const emojiStrip = require("emoji-strip");
+const fs = require("fs");
+const path = require("path");
 const Twitter = require("twitter");
+const html_entities_1 = require("html-entities");
 const loggers_1 = require("./loggers");
 const utils_1 = require("./utils");
+const wiki_1 = require("./wiki");
+exports.keywordMap = {
+    'ガチャ予告': '卡池',
+    '一日一回無料ガチャ': '卡池',
+    'イベント開催決定': '活动',
+    'タワー.*追加': '新塔',
+    '(生放送|配信)予告': '生放',
+    'メンテナンス予告': '维护',
+    '(育成応援|お仕事).*開催': '工作',
+    '新曲一部公開': '新曲',
+    'キャラクター紹介': '组合',
+    '今後のアップデート情報': '计划',
+    '(?<!今後の)アップデート情報': '改修',
+};
+exports.recurringKeywords = [
+    '(育成応援|お仕事).*開催', 'イベント開催決定', '無料ガチャ',
+    'アップデート情報', 'キャラクター紹介',
+    '(生放送|配信)予告', 'メンテナンス予告'
+];
+const xmlEntities = new html_entities_1.XmlEntities();
+const processTweetBody = (tweet) => {
+    const urls = tweet.entities.urls ? tweet.entities.urls.map(url => url.expanded_url) : [];
+    const [title, body] = emojiStrip(xmlEntities.decode(tweet.full_text))
+        .replace(/(?<=\n|^)(.*)(?:\: |：)(https?:\/\/.*?)(?=\s|$)/g, '[$2 $1]')
+        .replace(/https?:\/\/.*?(?=\s|$)/g, () => urls.length ? urls.splice(0, 1)[0] : '')
+        .replace(/(?<=\n|^)[＼／]\n/g, '')
+        .replace(/((?<=\s)#.*?\s+)+$/g, '')
+        .trim()
+        .match(/(.*?)\n\n(.*)/s).slice(1);
+    const date = new Date(tweet.created_at);
+    const formatedDate = [[date.getFullYear(), 4], [date.getMonth(), 2], [date.getDate(), 2]]
+        .map(([value, padLength]) => value.toString().padStart(padLength, '0'))
+        .join('');
+    const pageTitle = title.replace(/[【】＼／]/g, '') + `${exports.recurringKeywords.some(keyword => new RegExp(keyword).exec(title)) ? `-${formatedDate}` : ''}`;
+    return { title, body, pageTitle, date: formatedDate };
+};
+exports.processTweetBody = processTweetBody;
+const parseAction = (action) => {
+    if (!action || !Object.keys(action))
+        return '(无)';
+    return `\n
+  标题：${action.title}
+  操作：${action.new ? '新建' : '更新'}
+  媒体：${action.mediafiles.map((fileName, index) => `\n    ${index + 1}. ${fileName}`).join('') || '(无)'}
+  链接：${action.pageid ? `https://wiki.biligame.com/idolypride/index.php?curid=${action.pageid}` : '(无)'}
+  `;
+};
+exports.parseAction = parseAction;
 const TWITTER_EPOCH = 1288834974657;
 const snowflake = (epoch) => Number.isNaN(epoch) ? undefined :
     utils_1.BigNumOps.lShift(String(epoch - 1 - TWITTER_EPOCH), 22);
 exports.snowflake = snowflake;
-const logger = loggers_1.getLogger('twitter');
-let queryByRegExp = (username, regexp, queryThreshold, until) => Promise.resolve(null);
-exports.queryByRegExp = queryByRegExp;
+const logger = (0, loggers_1.getLogger)('twitter');
 class default_1 {
     constructor(opt) {
-        this.lastQueries = {};
-        this.queryUser = (username) => this.client.get('users/show', { screen_name: username })
-            .then((user) => user.screen_name);
-        this.queryTimelineReverse = (conf) => {
-            if (!conf.since)
-                return this.queryTimeline(conf);
-            const count = conf.count;
-            const maxID = conf.until;
-            conf.count = undefined;
-            const until = () => utils_1.BigNumOps.min(maxID, utils_1.BigNumOps.plus(conf.since, String(7 * 24 * 3600 * 1000 * Math.pow(2, 22))));
-            conf.until = until();
-            const promise = (tweets) => this.queryTimeline(conf).then(newTweets => {
-                tweets = newTweets.concat(tweets);
-                conf.since = conf.until;
-                conf.until = until();
-                if (tweets.length >= count ||
-                    utils_1.BigNumOps.compare(conf.since, conf.until) >= 0) {
-                    return tweets.slice(-count);
-                }
-                return promise(tweets);
+        this.launch = () => {
+            this.publisher.login(this.wikiSessionCookie).then(() => {
+                setTimeout(this.work, this.workInterval * 1000);
             });
-            return promise([]);
         };
-        this.queryTimeline = ({ username, count, since, until, noreps, norts }) => {
-            username = username.replace(/^@?(.*)$/, '@$1');
-            logger.info(`querying timeline of ${username} with config: ${JSON.stringify(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, (count && { count })), (since && { since })), (until && { until })), (noreps && { noreps })), (norts && { norts })))}`);
-            const fetchTimeline = (config = {
-                screen_name: username.slice(1),
-                trim_user: true,
-                exclude_replies: noreps !== null && noreps !== void 0 ? noreps : true,
-                include_rts: !(norts !== null && norts !== void 0 ? norts : false),
-                since_id: since,
-                max_id: until,
-                tweet_mode: 'extended',
-            }, tweets = []) => this.client.get('statuses/user_timeline', config)
-                .then((newTweets) => {
-                if (newTweets.length) {
-                    logger.debug(`fetched tweets: ${JSON.stringify(newTweets)}`);
-                    config.max_id = utils_1.BigNumOps.plus('-1', newTweets[newTweets.length - 1].id_str);
-                    logger.info(`timeline query of ${username} yielded ${newTweets.length} new tweets, next query will start at offset ${config.max_id}`);
-                    tweets.push(...newTweets);
+        this.work = () => {
+            const lock = this.lock;
+            if (this.workInterval < 1)
+                this.workInterval = 1;
+            const currentFeed = 'https://twitter.com/idolypride';
+            logger.debug(`pulling feed ${currentFeed}`);
+            const promise = new Promise(resolve => {
+                let config;
+                let endpoint;
+                const match = /https:\/\/twitter.com\/([^\/]+)/.exec(currentFeed);
+                if (match) {
+                    config = {
+                        screen_name: match[1],
+                        exclude_replies: false,
+                        tweet_mode: 'extended',
+                    };
+                    endpoint = 'statuses/user_timeline';
+                }
+                if (endpoint) {
+                    const offset = lock.offset;
+                    if (offset > 0)
+                        config.since_id = offset;
+                    const getMore = (lastTweets = []) => this.client.get(endpoint, config, (error, tweets) => {
+                        if (error) {
+                            if (error instanceof Array && error.length > 0 && error[0].code === 34) {
+                                logger.warn(`error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
+                                lock.subscribers.forEach(subscriber => {
+                                    logger.info(`sending notfound message of ${currentFeed} to ${JSON.stringify(subscriber)}`);
+                                    this.bot.sendTo(subscriber, `错误：链接 ${currentFeed} 指向的用户或列表不存在。`).catch();
+                                });
+                            }
+                            else {
+                                logger.error(`unhandled error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
+                            }
+                        }
+                        if (!(tweets instanceof Array) || tweets.length === 0)
+                            return resolve(lastTweets);
+                        if (offset <= 0)
+                            return resolve(lastTweets.concat(tweets));
+                        config.max_id = utils_1.BigNumOps.plus(tweets.slice(-1)[0].id_str, '-1');
+                        getMore(lastTweets.concat(tweets));
+                    });
+                    getMore();
+                }
+            });
+            promise.then((tweets) => {
+                logger.debug(`api returned ${JSON.stringify(tweets)} for feed ${currentFeed}`);
+                const updateDate = () => lock.updatedAt = new Date().toString();
+                if (tweets.length === 0) {
+                    updateDate();
+                    return;
                 }
-                if (!newTweets.length || count === undefined || tweets.length >= count) {
-                    logger.info(`timeline query of ${username} finished successfully, ${tweets.length} tweets have been fetched`);
-                    return tweets.slice(0, count);
+                const topOfFeed = tweets[0].id_str;
+                const updateOffset = () => lock.offset = topOfFeed;
+                if (lock.offset === '-1') {
+                    updateOffset();
+                    return;
                 }
-                return fetchTimeline(config, tweets);
+                if (lock.offset === '0')
+                    tweets.splice(1);
+                return (0, utils_1.chainPromises)(tweets.reverse().map(tweet => () => {
+                    const match = /(.*?)\n\n(.*)/s.exec(tweet.full_text);
+                    if (!match)
+                        return Promise.resolve({});
+                    for (const keyword in exports.keywordMap) {
+                        if (new RegExp(keyword).exec(match[1])) {
+                            const tweetUrl = `${currentFeed}/status/${tweet.id_str}`;
+                            logger.info(`working on ${tweetUrl}`);
+                            return this.publisher.post(tweet, exports.keywordMap[keyword])
+                                .then(action => {
+                                if (action.result === 'Success') {
+                                    this.lock.lastActions.push(action);
+                                    logger.info(`successfully posted content of ${tweetUrl} to bwiki, link:`);
+                                    logger.info(`https://wiki.biligame.com/idolypride/index.php?curid=${action.pageid}`);
+                                    const message = `已更新如下页面：${(0, exports.parseAction)(action)}`;
+                                    return Promise.all(this.lock.subscribers.map(subscriber => this.bot.sendTo(subscriber, message)));
+                                }
+                            })
+                                .then(updateDate).then(updateOffset);
+                        }
+                    }
+                }));
+            })
+                .then(() => {
+                let timeout = this.workInterval * 1000;
+                if (timeout < 1000)
+                    timeout = 1000;
+                fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
+                setTimeout(() => {
+                    this.work();
+                }, timeout);
             });
-            return fetchTimeline();
         };
         this.client = new Twitter({
             consumer_key: opt.consumerKey,
@@ -69,35 +167,12 @@ class default_1 {
             access_token_key: opt.accessTokenKey,
             access_token_secret: opt.accessTokenSecret,
         });
-        exports.queryByRegExp = (username, regexp, queryThreshold, until) => {
-            logger.info(`searching timeline of @${username} for matches of ${regexp}...`);
-            const normalizedUsername = username.toLowerCase().replace(/^@/, '');
-            const queryKey = `${normalizedUsername}:${regexp.toString()}`;
-            const isOld = (then, threshold = 360) => {
-                if (!then)
-                    return true;
-                return utils_1.BigNumOps.compare(exports.snowflake(Date.now() - threshold * 1000), then) >= 0;
-            };
-            if (queryKey in this.lastQueries && !isOld(this.lastQueries[queryKey].id)) {
-                const { match, id } = this.lastQueries[queryKey];
-                logger.info(`found match ${JSON.stringify(match)} from cached tweet of id ${id}`);
-                return Promise.resolve(match);
-            }
-            return this.queryTimeline({ username, norts: true, until })
-                .then(tweets => {
-                const found = tweets.find(tweet => regexp.test(tweet.full_text));
-                if (found) {
-                    const match = regexp.exec(found.full_text);
-                    this.lastQueries[queryKey] = { match, id: found.id_str };
-                    logger.info(`found match ${JSON.stringify(match)} in tweet of id ${found.id_str} from timeline`);
-                    return match;
-                }
-                const last = tweets.slice(-1)[0].id_str;
-                if (isOld(last, queryThreshold))
-                    return null;
-                return exports.queryByRegExp(username, regexp, queryThreshold, last);
-            });
-        };
+        this.bot = opt.bot;
+        this.publisher = new wiki_1.default(opt.lock);
+        this.lockfile = opt.lockfile;
+        this.lock = opt.lock;
+        this.workInterval = opt.workInterval;
+        this.wikiSessionCookie = opt.wikiSessionCookie;
     }
 }
 exports.default = default_1;

dist/utils.js

@@ -1,39 +1,9 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Arr = exports.BigNumOps = exports.rawRegExp = exports.customError = exports.CustomError = exports.neverResolves = exports.chainPromises = void 0;
-const CallableInstance = require("callable-instance");
+exports.BigNumOps = exports.chainPromises = void 0;
 const chainPromises = (lazyPromises, reducer = (lp1, lp2) => (p) => lp1(p).then(lp2), initialValue) => lazyPromises.reduce(reducer, p => Promise.resolve(p))(initialValue);
 exports.chainPromises = chainPromises;
-const neverResolves = () => new Promise(() => undefined);
-exports.neverResolves = neverResolves;
-class CustomErrorConstructor extends CallableInstance {
-    constructor(name) {
-        super('getError');
-        Object.defineProperty(this.constructor, 'name', { value: name });
-    }
-    getError(message) { return new CustomError(this.constructor.name, message); }
-}
-class CustomError extends Error {
-    constructor(name, message) {
-        super(message);
-        this.name = name;
-        Object.defineProperty(this.constructor, 'name', { value: name });
-    }
-}
-exports.CustomError = CustomError;
-const customError = (name) => new CustomErrorConstructor(name);
-exports.customError = customError;
-const chunkArray = (arr, size) => {
-    const noOfChunks = Math.ceil(size && arr.length / size);
-    const res = Array(noOfChunks);
-    for (let [i, j] = [0, 0]; i < noOfChunks; i++) {
-        res[i] = arr.slice(j, j += size);
-    }
-    return res;
-};
-const rawRegExp = (...args) => RegExp(String.raw(...args));
-exports.rawRegExp = rawRegExp;
-const splitBigNumAt = (num, at) => num.replace(exports.rawRegExp `^([+-]?)(\d+)(\d{${at}})$`, '$1$2,$1$3')
+const splitBigNumAt = (num, at) => num.replace(RegExp(String.raw `^([+-]?)(\d+)(\d{${at}})$`), '$1$2,$1$3')
     .replace(/^([^,]*)$/, '0,$1').split(',')
     .map(Number);
 const bigNumPlus = (num1, num2) => {
@@ -79,4 +49,3 @@ exports.BigNumOps = {
     lShift: bigNumLShift,
     parse: parseBigNum,
 };
-exports.Arr = { chunk: chunkArray };

dist/wiki.js

@@ -0,0 +1,193 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const axios_1 = require("axios");
+const fetchCookie = require("fetch-cookie");
+const fs_1 = require("fs");
+const mediawiki2_1 = require("mediawiki2");
+const node_fetch_1 = require("node-fetch");
+const playwright_1 = require("playwright");
+const loggers_1 = require("./loggers");
+const twitter_1 = require("./twitter");
+const logger = (0, loggers_1.getLogger)('wiki');
+const baseUrl = 'https://wiki.biligame.com/idolypride';
+class default_1 {
+    constructor(lock) {
+        this.login = (sessdata) => playwright_1.firefox.launch().then(browser => {
+            const jar = this.bot.cookieJar;
+            return browser.newPage().then(page => page.context().addCookies([{
+                    name: 'SESSDATA',
+                    value: sessdata,
+                    domain: '.biligame.com',
+                    path: '/',
+                }])
+                .then(() => page.route('**/*.{png,jpg,jpeg,gif}', route => route.abort()))
+                .then(() => page.route('*://*.baidu.com/**', route => route.abort()))
+                .then(() => page.goto(`${baseUrl}/index.php?curid=2`, { waitUntil: 'networkidle' }))
+                .then(() => { logger.info('logging in via browser...'); return page.context().cookies(); })
+                .then(cookies => {
+                const uidIndex = cookies.findIndex(cookie => cookie.name === 'gamecenter_wiki_UserName');
+                if (!uidIndex)
+                    throw new Error('auth error');
+                return Promise.all(cookies.map(({ name, value, domain, path }) => jar.setCookie(`${name}=${value}; Domain=${domain}; Path=${path}`, baseUrl))).then(() => cookies[uidIndex].value);
+            })
+                .then(uid => {
+                logger.info(`finished logging in via browser, wiki username: ${uid}`);
+                this.bot.fetch = fetchCookie(node_fetch_1.default, jar);
+                return browser.close();
+            })
+                .catch((err) => browser.close().then(() => {
+                logger.fatal(`error logging in via browser, error: ${err}`);
+                process.exit(0);
+            })));
+        });
+        this.fetchMedia = (url) => new Promise((resolve, reject) => {
+            logger.info(`fetching ${url}`);
+            const fetch = () => (0, axios_1.default)({
+                method: 'get',
+                url,
+                responseType: 'arraybuffer',
+                timeout: 150000,
+            }).then(res => {
+                if (res.status === 200) {
+                    logger.info(`successfully fetched ${url}`);
+                    resolve(res.data);
+                }
+                else {
+                    logger.error(`failed to fetch ${url}: ${res.status}`);
+                    reject();
+                }
+            }).catch(err => {
+                logger.error(`failed to fetch ${url}: ${err instanceof Error ? err.message : err}`);
+                logger.info(`trying to fetch ${url} again...`);
+                fetch();
+            });
+            fetch();
+        }).then(data => {
+            var _a;
+            return (([_, filename, ext]) => {
+                if (ext) {
+                    const mediaFileName = `${filename}.${ext}`;
+                    (0, fs_1.writeFileSync)(mediaFileName, Buffer.from(data));
+                    return mediaFileName;
+                }
+                logger.warn('unable to find MIME type of fetched media, failing this fetch');
+                throw Error();
+            })((_a = /([^\/]*)\?format=([a-z]+)&/.exec(url)) !== null && _a !== void 0 ? _a : /([^\/]*)\.([^?]+)/.exec(url));
+        });
+        this.uploadMediaItems = (tweet, fileNamePrefix, indexOffset = 0) => {
+            const mediaItems = [];
+            if (tweet.extended_entities) {
+                tweet.extended_entities.media.forEach((media, index) => {
+                    let url;
+                    if (media.type === 'photo') {
+                        url = media.media_url_https.replace(/\.([a-z]+)$/, '?format=$1') + '&name=orig';
+                    }
+                    else {
+                        url = media.video_info.variants
+                            .filter(variant => variant.bitrate !== undefined)
+                            .sort((var1, var2) => var2.bitrate - var1.bitrate)
+                            .map(variant => variant.url)[0];
+                    }
+                    const mediaPromise = this.fetchMedia(url)
+                        .then(mediaFileName => {
+                        const filename = `${fileNamePrefix}${indexOffset + index + 1}.${mediaFileName.split('.')[1]}`;
+                        logger.info(`uploading ${url} as ${filename}...`);
+                        return this.bot.simpleUpload({
+                            file: mediaFileName,
+                            filename,
+                        }).then(() => filename);
+                    });
+                    mediaItems.push(mediaPromise);
+                });
+            }
+            return Promise.all(mediaItems);
+        };
+        this.appendMedia = (tweet, genre, indexOffset) => {
+            const { pageTitle } = (0, twitter_1.processTweetBody)(tweet);
+            return this.uploadMediaItems(tweet, `公告-${genre}-${pageTitle}-`, indexOffset)
+                .then(fileNames => {
+                logger.info(`updating page 公告/${pageTitle}...`);
+                return this.bot.edit({
+                    title: `公告/${pageTitle}`,
+                    appendtext: `${fileNames.map(fileName => `[[文件:${fileName}|无框|左]]\n`).join('')}`,
+                    bot: true,
+                    notminor: true,
+                    nocreate: true,
+                })
+                    .then(({ new: isNewPost, newtimestamp, pageid, result, title }) => ({
+                    pageid,
+                    title,
+                    new: isNewPost,
+                    mediafiles: fileNames,
+                    result,
+                    timestamp: new Date(newtimestamp).toString(),
+                }))
+                    .catch(error => {
+                    logger.error(`error updating page, error: ${error}`);
+                    return {
+                        pageid: undefined,
+                        title: `公告/${pageTitle}`,
+                        new: undefined,
+                        mediafiles: [],
+                        result: 'Failed',
+                        timestamp: undefined,
+                    };
+                });
+            });
+        };
+        this.post = (tweet, genre) => {
+            const { title, body, pageTitle, date } = (0, twitter_1.processTweetBody)(tweet);
+            const sameTitleAction = this.lock.lastActions.find(action => action.title === title);
+            if (sameTitleAction)
+                return this.appendMedia(tweet, genre, sameTitleAction.mediafiles.length);
+            return this.uploadMediaItems(tweet, `公告-${genre}-${pageTitle}-`)
+                .then(fileNames => {
+                logger.info(`creating page 公告/${pageTitle}...`);
+                return this.bot.edit({
+                    title: `公告/${pageTitle}`,
+                    basetimestamp: new Date(),
+                    text: `{{文章戳
+|文章上级页面=公告
+|子类别=${genre}
+|时间=${date}
+|作者=IDOLY PRIDE
+|是否原创=否
+|来源=[https://twitter.com/idolypride IDOLY PRIDE]
+|原文地址=[https://twitter.com/idolypride/status/${tweet.id_str} ${pageTitle}]
+}}
+====${title}====
+<poem>
+${body}
+</poem>
+${fileNames.map(fileName => `[[文件:${fileName}|无框|左]]`).join('\n')}
+`,
+                    bot: true,
+                    notminor: true,
+                    createonly: true,
+                })
+                    .then(({ new: isNewPost, newtimestamp, pageid, result, title }) => ({
+                    pageid,
+                    title,
+                    new: isNewPost,
+                    mediafiles: fileNames,
+                    result,
+                    timestamp: new Date(newtimestamp).toString(),
+                }))
+                    .catch(error => {
+                    logger.error(`error creating page, error: ${error}`);
+                    return {
+                        pageid: undefined,
+                        title: `公告/${pageTitle}`,
+                        new: undefined,
+                        mediafiles: [],
+                        result: 'Failed',
+                        timestamp: undefined,
+                    };
+                });
+            });
+        };
+        this.bot = new mediawiki2_1.MWBot(`${baseUrl}/api.php`);
+        this.lock = lock;
+    }
+}
+exports.default = default_1;

package.json

@@ -1,10 +1,10 @@
 {
-  "name": "@CL-Jeremy/mirai-twitter-bot",
-  "version": "0.4.0",
-  "description": "Mirai Twitter Bot",
+  "name": "gocqhttp-ipwiki-bot",
+  "version": "0.0.1",
+  "description": "GoCQHTTP IPWIKI Bot",
   "main": "./dist/main.js",
   "bin": {
-    "mirai-twitter-bot": "./dist/main.js"
+    "cq-ipwiki-bot": "./dist/main.js"
   },
   "repository": {
     "type": "git",
@@ -26,28 +26,31 @@
   },
   "homepage": "https://github.com/CL-Jeremy/mirai-twitter-bot",
   "scripts": {
+    "preinstall": "npx force-resolutions",
     "build": "rm -rf dist && npx tsc --outDir d && mv d/src dist && rm -rf d",
     "lint": "npx eslint --fix --ext .ts ./"
   },
   "dependencies": {
     "callable-instance": "^2.0.0",
     "command-line-usage": "^5.0.5",
+    "emoji-strip": "git+https://github.com/ivangil-dev/emoji-strip.git",
+    "fetch-cookie": "^2.0.0",
     "html-entities": "^1.3.1",
     "koishi": "^3.10.0",
     "koishi-adapter-onebot": "^3.0.8",
     "log4js": "^6.3.0",
-    "playwright": "^1.9.1",
+    "mediawiki2": "0.0.3",
+    "playwright": "^1.18.1",
     "pngjs": "^5.0.0",
     "read-all-stream": "^3.1.0",
     "sha1": "^1.1.1",
+    "tough-cookie": "^4.0.0",
     "twitter": "^1.7.1",
     "typescript": "^4.2.3"
   },
   "devDependencies": {
     "@types/command-line-usage": "^5.0.1",
     "@types/node": "^14.14.22",
-    "@types/pngjs": "^3.4.2",
-    "@types/puppeteer": "^1.5.0",
     "@types/twitter": "^1.7.0",
     "@typescript-eslint/eslint-plugin": "^4.22.0",
     "@typescript-eslint/parser": "^4.22.0",
@@ -56,7 +59,12 @@
     "eslint-plugin-jsdoc": "^32.3.1",
     "eslint-plugin-prefer-arrow": "^1.2.3",
     "eslint-plugin-react": "^7.23.2",
+    "npm-force-resolutions": "0.0.10",
     "tslint-config-prettier": "^1.13.0",
     "twitter-d": "^0.4.0"
+  },
+  "resolutions": {
+    "fetch-cookie": "^2.0.0",
+    "tough-cookie": "^4.0.0"
   }
 }

src/command.ts

@@ -2,8 +2,8 @@
 /* eslint-disable @typescript-eslint/member-delimiter-style */
 /* eslint-disable prefer-arrow/prefer-arrow-functions */
 
-import { Message } from './koishi';
-import { queryByRegExp } from './twitter';
+import { relativeDate } from './datetime';
+import { parseAction } from './twitter';
 
 function parseCmd(message: string): {
   cmd: string;
@@ -28,23 +28,42 @@ function parseCmd(message: string): {
   };
 }
 
-function view(_chat: IChat, args: string[], reply: (msg: string) => any): void {
-  let query = Number(args[0]);
-  if (args[0] && !Number.isInteger(query)) return reply('查询格式有误。\n格式：/nanatsuu_view [〈整数〉]');
-  queryByRegExp('t7s_staff', /の『それゆけ！ナナスタ☆通信』第(\d+)話はこちら！/, 3600 * 24 * 7)
-    .then(match => {
-      if (!match) throw Error();
-      if (!args[0]) query = Number(match[1]);
-      if (query < 0) query += Number(match[1]);
-      if (query < 0 || query > Number(match[1])) {
-        return reply(`查询取值范围有误。当前可用的取值范围：${- Number(match[1])}～${Number(match[1])}`);
-      }
-      reply(`第 ${query} 话：\n` +
-        Message.Image(`https://d2n19nac4w0gh6.cloudfront.net/resource/images/webview/comic/story/comic_${
-          String(query).padStart(3, '0')
-        }.jpg`)
-      );
-    }).catch((err: Error) => reply(`查询失败，请稍后重试。${err.message ? `原因：${err}`: ''}`));
+function sub(chat: IChat, args: string[], reply: (msg: string) => any,
+  lock: ILock, lockfile: string
+): void {
+  if (chat.chatType === ChatType.Temp) {
+    return reply('请先添加机器人为好友。');
+  }
+  const index = lock.subscribers.findIndex(({chatID, chatType}) =>
+    chat.chatID === chatID && chat.chatType === chatType
+  );
+  if (index > -1) return reply('此聊天已订阅 IDOLY PRIDE BWIKI 更新提醒。');
+  lock.subscribers.push(chat);
+  reply('已为此聊天订阅 IDOLY PRIDE BWIKI 更新提醒。');
 }
 
-export { parseCmd, view };
+function unsub(chat: IChat, args: string[], reply: (msg: string) => any,
+  lock: ILock, lockfile: string
+): void {
+  if (chat.chatType === ChatType.Temp) {
+    return reply('请先添加机器人为好友。');
+  }
+  const index = lock.subscribers.findIndex(({chatID, chatType}) =>
+    chat.chatID === chatID && chat.chatType === chatType
+  );
+  if (index === -1) return reply('此聊天未订阅 IDOLY PRIDE BWIKI 更新提醒。');
+  lock.subscribers.splice(index, 1);
+  reply('已为此聊天退订 IDOLY PRIDE BWIKI 更新提醒。');
+}
+
+function status(chat: IChat, _: string[], reply: (msg: string) => any, lock: ILock): void {
+  const lastAction = lock.lastActions.sort(
+    (a1, a2) => Date.parse(a2.timestamp) - Date.parse(a1.timestamp)
+  )[0];
+  reply(`IDOLY PRIDE 官方推特追踪情况：
+上次更新时间：${relativeDate(lastAction.timestamp || '')}
+上次更新内容：${parseAction(lastAction)}
+上次检查时间：${relativeDate(lock.updatedAt || '')}`);
+}
+
+export { parseCmd, sub, unsub, status };

src/datetime.js

@@ -0,0 +1,41 @@
+function relativeDate(dtstr) {
+  if (!dtstr) return '暂无数据';
+  const dt = new Date(dtstr);
+  const dateTimeStamp = dt.getTime();
+  const second = 1000;
+  const minute = second * 60;
+  const hour = minute * 60;
+  const day = hour * 24;
+  const month = day * 30;
+  const now = new Date().getTime();
+  const diffValue = now - dateTimeStamp;
+  if (diffValue < 0) {
+    return;
+  }
+  const monthC = diffValue / month;
+  const weekC = diffValue / (7 * day);
+  const dayC = diffValue / day;
+  const hourC = diffValue / hour;
+  const minC = diffValue / minute;
+  const secC = diffValue / second;
+  let result;
+  if (monthC > 12) {
+    const y = dt.getFullYear() + ' 年';
+    const m = dt.getMonth() + 1 + ' 月';
+    const d = dt.getDate() + ' 日';
+    result = [y, m, d].join(' ');
+  } else if (monthC >= 1) {
+    result = '' + Math.floor(monthC) + ' 个月前';
+  } else if (weekC >= 1) {
+    result = '' + Math.floor(weekC) + ' 周前';
+  } else if (dayC >= 1) {
+    result = '' + Math.floor(dayC) + ' 天前';
+  } else if (hourC >= 1) {
+    result = '' + Math.floor(hourC) + ' 小时前';
+  } else if (minC >= 1) {
+    result = '' + Math.floor(minC) + ' 分钟前';
+  } else result = '' + Math.floor(secC) + ' 秒前';
+  return result;
+}
+
+export { relativeDate };

src/emoji-strip.d.ts

@@ -0,0 +1,5 @@
+declare module 'emoji-strip' {
+  const emojiStrip: (arg: string) => string;
+  namespace emojiStrip { }
+  export = emojiStrip;
+}

src/koishi.ts

@@ -2,7 +2,7 @@ import { App, Bot, segment, Session, sleep } from 'koishi';
 import 'koishi-adapter-onebot';
 import { Message as CQMessage, SenderInfo } from 'koishi-adapter-onebot';
 
-import { parseCmd, view } from './command';
+import { parseCmd } from './command';
 import { getLogger } from './loggers';
 import { chainPromises } from './utils';
 
@@ -15,6 +15,9 @@ interface IQQProps {
   host: string;
   port: number;
   bot_id: number;
+  status(chat: IChat, args: string[], replyfn: (msg: string) => any): void;
+  sub(chat: IChat, args: string[], replyfn: (msg: string) => any): void;
+  unsub(chat: IChat, args: string[], replyfn: (msg: string) => any): void;
 }
 
 const cqUrlFix = (factory: segment.Factory<string | ArrayBuffer | Buffer>) =>
@@ -193,12 +196,24 @@ export default class {
           .catch(error => { logger.error(`error replying to message from ${userString}, error: ${error}`); });
       };
       switch (cmdObj.cmd) {
-        case 'nanatsuu_view':
-          view(chat, cmdObj.args, reply);
+        case 'ipwikistatus_sub':
+          this.botInfo.sub(chat, cmdObj.args, reply);
+          break;
+        case 'ipwikistatus_unsub':
+          this.botInfo.unsub(chat, cmdObj.args, reply);
+          break;
+        case 'ipwikistatus':
+          this.botInfo.status(chat, cmdObj.args, reply);
           break;
         case 'help':
-          if (cmdObj.args.length === 0) {
-            reply('Nanasta 通信搬运机器人：\n/nanatsuu_view [〈整数〉] - 查看最新或指定的 Nanasta 通信话数');
+          if (cmdObj.args[1] === 'ipwikistatus') {
+            reply(`IDOLY PRIDE BWIKI 机器人：
+/ipwikistatus - 查询当前状态
+/ipwikistatus_sub - 订阅状态更新
+/ipwikistatus_unsub - 退订状态更新\
+${chat.chatType === ChatType.Temp ?
+    '\n（当前游客模式下无法使用订阅功能，请先添加本账号为好友。）' : ''
+}`);
           }
       }
     }, true);

src/main.ts

@@ -9,19 +9,20 @@ import * as exampleConfig from '../config.example.json';
 import { getLogger, setLogLevels } from './loggers';
 import QQBot from './koishi';
 import Worker from './twitter';
+import { status, sub, unsub } from './command';
 
 const logger = getLogger();
 
 const sections: commandLineUsage.Section[] = [
   {
-    header: 'GoCQHTTP Nana Bot',
-    content: 'The QQ Bot that does stuff.',
+    header: 'GoCQHTTP IPWIKI Bot',
+    content: 'The QQ Bot that updates Idoly Pride BWIKI.',
   },
   {
     header: 'Synopsis',
     content: [
-      '$ cq-nana-bot {underline config.json}',
-      '$ cq-nana-bot {bold --help}',
+      '$ cq-ipwiki-bot {underline config.json}',
+      '$ cq-ipwiki-bot {bold --help}',
     ],
   },
   {
@@ -56,7 +57,7 @@ try {
 
 const requiredFields = [
   'twitter_consumer_key', 'twitter_consumer_secret', 'twitter_access_token_key', 'twitter_access_token_secret',
-  'cq_bot_qq',
+  'cq_bot_qq', 'bilibili_cookie_sessdata'
 ];
 
 const warningFields = [
@@ -64,7 +65,7 @@ const warningFields = [
 ];
 
 const optionalFields = [
-  'loglevel',
+  'lockfile', 'work_interval', 'loglevel',
 ].concat(warningFields);
 
 if (requiredFields.some((value) => config[value] === undefined)) {
@@ -81,18 +82,61 @@ optionalFields.forEach(key => {
 
 setLogLevels(config.loglevel);
 
+let lock: ILock;
+if (fs.existsSync(path.resolve(config.lockfile))) {
+  try {
+    lock = JSON.parse(fs.readFileSync(path.resolve(config.lockfile), 'utf8')) as ILock;
+  } catch (err) {
+    logger.error(`Failed to parse lockfile ${config.lockfile}: `, err);
+    lock = {
+      offset: '0',
+      lastActions: [],
+      subscribers: [],
+      updatedAt: undefined,
+    };
+  }
+  fs.access(path.resolve(config.lockfile), fs.constants.W_OK, err => {
+    if (err) {
+      logger.fatal(`cannot write lockfile ${path.resolve(config.lockfile)}, permission denied`);
+      process.exit(1);
+    }
+  });
+} else {
+  lock = {
+    offset: '0',
+    lastActions: [],
+    subscribers: [],
+    updatedAt: undefined,
+  };
+  try {
+    fs.writeFileSync(path.resolve(config.lockfile), JSON.stringify(lock));
+  } catch (err) {
+    logger.fatal(`cannot write lockfile ${path.resolve(config.lockfile)}, permission denied`);
+    process.exit(1);
+  }
+}
+
 const qq = new QQBot({
   access_token: config.cq_access_token,
   host: config.cq_ws_host,
   port: config.cq_ws_port,
   bot_id: config.cq_bot_qq,
+  status: (c, a, cb) => status(c, a, cb, lock),
+  sub: (c, a, cb) => sub(c, a, cb, lock, config.lockfile),
+  unsub: (c, a, cb) => unsub(c, a, cb, lock, config.lockfile),
 });
 
-new Worker({
+const worker = new Worker({
   consumerKey: config.twitter_consumer_key,
   consumerSecret: config.twitter_consumer_secret,
   accessTokenKey: config.twitter_access_token_key,
   accessTokenSecret: config.twitter_access_token_secret,
+  wikiSessionCookie: config.bilibili_cookie_sessdata,
+  lock,
+  lockfile: config.lockfile,
+  workInterval: config.work_interval,
+  bot: qq,
 });
+worker.launch();
 
 qq.connect();

src/model.d.ts

@@ -22,15 +22,17 @@ interface ITempChat {
 type IChat = IPrivateChat | IGroupChat | ITempChat;
 
 interface ILock {
-  workon: number;
-  feed: string[];
-  threads: {
-    [key: string]:
-    {
-      id: number,
-      offset: string,
-      updatedAt: string,
-      subscribers: IChat[],
-    },
-  };
+  offset: string;
+  lastActions: WikiEditResult[];
+  subscribers: IChat[];
+  updatedAt: string;
+}
+
+interface WikiEditResult {
+  pageid: number;
+  title: string;
+  new: boolean;
+  result: string;
+  mediafiles: string[];
+  timestamp: string;
 }

src/twitter.ts

@@ -1,10 +1,21 @@
+import * as emojiStrip from 'emoji-strip';
+import * as fs from 'fs';
+import * as path from 'path';
 import * as Twitter from 'twitter';
 import TwitterTypes from 'twitter-d';
+import { XmlEntities } from 'html-entities';
 
+import QQBot from './koishi';
 import { getLogger } from './loggers';
-import { BigNumOps } from './utils';
+import { BigNumOps, chainPromises } from './utils';
+import Wiki from './wiki';
 
 interface IWorkerOption {
+  lock: ILock;
+  lockfile: string;
+  bot: QQBot,
+  workInterval: number;
+  wikiSessionCookie: string;
   consumerKey: string;
   consumerSecret: string;
   accessTokenKey: string;
@@ -20,6 +31,57 @@ export interface ITimelineQueryConfig {
   norts?: boolean;
 }
 
+export const keywordMap = {
+  'ガチャ予告': '卡池',
+  '一日一回無料ガチャ': '卡池',
+  'イベント開催決定': '活动',
+  'タワー.*追加': '新塔',
+  '(生放送|配信)予告': '生放',
+  'メンテナンス予告': '维护',
+  '(育成応援|お仕事).*開催': '工作',
+  '新曲一部公開': '新曲',
+  'キャラクター紹介': '组合',
+  '今後のアップデート情報': '计划',
+  '(?<!今後の)アップデート情報': '改修',
+};
+
+export const recurringKeywords = [
+  '(育成応援|お仕事).*開催', 'イベント開催決定', '無料ガチャ',
+  'アップデート情報', 'キャラクター紹介',
+  '(生放送|配信)予告', 'メンテナンス予告'
+];
+
+const xmlEntities = new XmlEntities();
+
+export const processTweetBody = (tweet: Tweet) => {
+  const urls = tweet.entities.urls ? tweet.entities.urls.map(url => url.expanded_url) : [];
+  const [title, body] = emojiStrip(xmlEntities.decode(tweet.full_text))
+    .replace(/(?<=\n|^)(.*)(?:\: |：)(https?:\/\/.*?)(?=\s|$)/g, '[$2 $1]')
+    .replace(/https?:\/\/.*?(?=\s|$)/g, () => urls.length ? urls.splice(0, 1)[0] : '')
+    .replace(/(?<=\n|^)[＼／]\n/g, '')
+    .replace(/((?<=\s)#.*?\s+)+$/g, '')
+    .trim()
+    .match(/(.*?)\n\n(.*)/s).slice(1);
+    const date = new Date(tweet.created_at);
+    const formatedDate = [[date.getFullYear(), 4], [date.getMonth(), 2], [date.getDate(), 2]]
+      .map(([value, padLength]) => value.toString().padStart(padLength, '0'))
+      .join('');
+    const pageTitle = title.replace(/[【】＼／]/g, '') + `${recurringKeywords.some(
+      keyword => new RegExp(keyword).exec(title)
+    ) ? `-${formatedDate}` : ''}`;
+  return {title, body, pageTitle, date: formatedDate};
+};
+
+export const parseAction = (action: WikiEditResult) => {
+  if (!action || !Object.keys(action)) return '(无)';
+  return `\n
+  标题：${action.title}
+  操作：${action.new ? '新建' : '更新'}
+  媒体：${action.mediafiles.map((fileName, index) => `\n    ${index + 1}. ${fileName}`).join('') || '(无)'}
+  链接：${action.pageid ? `https://wiki.biligame.com/idolypride/index.php?curid=${action.pageid}` : '(无)'}
+  `;
+}
+
 const TWITTER_EPOCH = 1288834974657;
 export const snowflake = (epoch: number) => Number.isNaN(epoch) ? undefined :
   BigNumOps.lShift(String(epoch - 1 - TWITTER_EPOCH), 22);
@@ -39,13 +101,15 @@ interface ITweet extends TwitterTypes.Status {
 export type Tweet = ITweet;
 export type Tweets = ITweet[];
 
-export let queryByRegExp = (username: string, regexp: RegExp, queryThreshold?: number, until?: string) =>
-  Promise.resolve<RegExpExecArray>(null);
-
 export default class {
 
   private client: Twitter;
-  private lastQueries: {[key: string]: {match: RegExpExecArray, id: string}} = {};
+  private bot: QQBot;
+  private publisher: Wiki;
+  private lock: ILock;
+  private lockfile: string;
+  private workInterval: number;
+  private wikiSessionCookie: string;
 
   constructor(opt: IWorkerOption) {
     this.client = new Twitter({
@@ -54,101 +118,109 @@ export default class {
       access_token_key: opt.accessTokenKey,
       access_token_secret: opt.accessTokenSecret,
     });
-    queryByRegExp = (username, regexp, queryThreshold?, until?) => {
-      logger.info(`searching timeline of @${username} for matches of ${regexp}...`);
-      const normalizedUsername = username.toLowerCase().replace(/^@/, '');
-      const queryKey = `${normalizedUsername}:${regexp.toString()}`;
-      const isOld = (then: string, threshold = 360) => {
-        if (!then) return true;
-        return BigNumOps.compare(snowflake(Date.now() - threshold * 1000), then) >= 0;
-      };
-      if (queryKey in this.lastQueries && !isOld(this.lastQueries[queryKey].id)) {
-        const {match, id} = this.lastQueries[queryKey];
-        logger.info(`found match ${JSON.stringify(match)} from cached tweet of id ${id}`);
-        return Promise.resolve(match);
+    this.bot = opt.bot;
+    this.publisher = new Wiki(opt.lock);
+    this.lockfile = opt.lockfile;
+    this.lock = opt.lock;
+    this.workInterval = opt.workInterval;
+    this.wikiSessionCookie = opt.wikiSessionCookie;
+  }
+
+  public launch = () => {
+    this.publisher.login(this.wikiSessionCookie).then(() => {
+      setTimeout(this.work, this.workInterval * 1000)
+    });
+  }
+
+  public work = () => {
+    const lock = this.lock;
+    if (this.workInterval < 1) this.workInterval = 1;
+
+    const currentFeed = 'https://twitter.com/idolypride';
+    logger.debug(`pulling feed ${currentFeed}`);
+
+    const promise = new Promise(resolve => {
+      let config: {[key: string]: any};
+      let endpoint: string;
+      const match = /https:\/\/twitter.com\/([^\/]+)/.exec(currentFeed);
+      if (match) {
+        config = {
+          screen_name: match[1],
+          exclude_replies: false,
+          tweet_mode: 'extended',
+        };
+        endpoint = 'statuses/user_timeline';
       }
-      return this.queryTimeline({username, norts: true, until})
-        .then(tweets => {
-          const found = tweets.find(tweet => regexp.test(tweet.full_text));
-          if (found) {
-            const match = regexp.exec(found.full_text);
-            this.lastQueries[queryKey] = {match, id: found.id_str};
-            logger.info(`found match ${JSON.stringify(match)} in tweet of id ${found.id_str} from timeline`);
-            return match;
+
+      if (endpoint) {
+        const offset = lock.offset;
+        if (offset as unknown as number > 0) config.since_id = offset;
+        const getMore = (lastTweets: Tweets = []) => this.client.get(
+          endpoint, config, (error: {[key: string]: any}[], tweets: Tweets
+        ) => {
+          if (error) {
+            if (error instanceof Array && error.length > 0 && error[0].code === 34) {
+              logger.warn(`error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
+              lock.subscribers.forEach(subscriber => {
+                logger.info(`sending notfound message of ${currentFeed} to ${JSON.stringify(subscriber)}`);
+                this.bot.sendTo(subscriber, `错误：链接 ${currentFeed} 指向的用户或列表不存在。`).catch();
+              });
+            } else {
+              logger.error(`unhandled error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
+            }
           }
-          const last = tweets.slice(-1)[0].id_str;
-          if (isOld(last, queryThreshold)) return null;
-          return queryByRegExp(username, regexp, queryThreshold, last);
+          if (!(tweets instanceof Array) || tweets.length === 0) return resolve(lastTweets);
+          if (offset as unknown as number <= 0) return resolve(lastTweets.concat(tweets));
+          config.max_id = BigNumOps.plus(tweets.slice(-1)[0].id_str, '-1');
+          getMore(lastTweets.concat(tweets));
         });
-    };
-  }
-
-  public queryUser = (username: string) => this.client.get('users/show', {screen_name: username})
-    .then((user: FullUser) => user.screen_name);
-
-  public queryTimelineReverse = (conf: ITimelineQueryConfig) => {
-    if (!conf.since) return this.queryTimeline(conf);
-    const count = conf.count;
-    const maxID = conf.until;
-    conf.count = undefined;
-    const until = () => BigNumOps.min(maxID, BigNumOps.plus(conf.since, String(7 * 24 * 3600 * 1000 * 2 ** 22)));
-    conf.until = until();
-    const promise = (tweets: ITweet[]): Promise<ITweet[]> =>this.queryTimeline(conf).then(newTweets => {
-      tweets = newTweets.concat(tweets);
-      conf.since = conf.until;
-      conf.until = until();
-      if (
-        tweets.length >= count ||
-          BigNumOps.compare(conf.since, conf.until) >= 0
-      ) {
-        return tweets.slice(-count);
+        getMore();
       }
-      return promise(tweets);
     });
-    return promise([]);
-  };
 
-  public queryTimeline = (
-    { username, count, since, until, noreps, norts }: ITimelineQueryConfig
-  ) => {
-    username = username.replace(/^@?(.*)$/, '@$1');
-    logger.info(`querying timeline of ${username} with config: ${
-      JSON.stringify({
-        ...(count && {count}),
-        ...(since && {since}),
-        ...(until && {until}),
-        ...(noreps && {noreps}),
-        ...(norts && {norts}),
-      })}`);
-    const fetchTimeline = (
-      config = {
-        screen_name: username.slice(1),
-        trim_user: true,
-        exclude_replies: noreps ?? true,
-        include_rts: !(norts ?? false),
-        since_id: since,
-        max_id: until,
-        tweet_mode: 'extended',
-      },
-      tweets: ITweet[] = []
-    ): Promise<ITweet[]> => this.client.get('statuses/user_timeline', config)
-      .then((newTweets: ITweet[]) => {
-        if (newTweets.length) {
-          logger.debug(`fetched tweets: ${JSON.stringify(newTweets)}`);
-          config.max_id = BigNumOps.plus('-1', newTweets[newTweets.length - 1].id_str);
-          logger.info(`timeline query of ${username} yielded ${
-            newTweets.length
-          } new tweets, next query will start at offset ${config.max_id}`);
-          tweets.push(...newTweets);
-        }
-        if (!newTweets.length || count === undefined || tweets.length >= count) {
-          logger.info(`timeline query of ${username} finished successfully, ${
-            tweets.length
-          } tweets have been fetched`);
-          return tweets.slice(0, count);
+    promise.then((tweets: Tweets) => {
+      logger.debug(`api returned ${JSON.stringify(tweets)} for feed ${currentFeed}`);
+      
+      const updateDate = () => lock.updatedAt = new Date().toString();
+      if (tweets.length === 0) { updateDate(); return; }
+
+      const topOfFeed = tweets[0].id_str;
+      const updateOffset = () => lock.offset = topOfFeed;
+
+      if (lock.offset === '-1') { updateOffset(); return; }
+      if (lock.offset === '0') tweets.splice(1);
+
+      return chainPromises(tweets.reverse().map(tweet => () => {
+        const match = /(.*?)\n\n(.*)/s.exec(tweet.full_text);
+        if (!match) return Promise.resolve({});
+        for (const keyword in keywordMap) {
+          if (new RegExp(keyword).exec(match[1])) {
+            const tweetUrl = `${currentFeed}/status/${tweet.id_str}`;
+            logger.info(`working on ${tweetUrl}`);
+            return this.publisher.post(tweet, keywordMap[keyword])
+              .then(action => {
+                if (action.result === 'Success') {
+                  this.lock.lastActions.push(action);
+                  logger.info(`successfully posted content of ${tweetUrl} to bwiki, link:`);
+                  logger.info(`https://wiki.biligame.com/idolypride/index.php?curid=${action.pageid}`)
+                  const message = `已更新如下页面：${parseAction(action)}`;
+                  return Promise.all(
+                    this.lock.subscribers.map(subscriber => this.bot.sendTo(subscriber, message))
+                  );
+                }
+              })
+              .then(updateDate).then(updateOffset);
+          }
         }
-        return fetchTimeline(config, tweets);
+      }));
+    })
+      .then(() => {
+        let timeout = this.workInterval * 1000;
+        if (timeout < 1000) timeout = 1000;
+        fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
+        setTimeout(() => {
+          this.work();
+        }, timeout);
       });
-    return fetchTimeline();
   };
 }

src/utils.ts

@@ -1,43 +1,10 @@
-import * as CallableInstance from 'callable-instance';
-
 export const chainPromises = <T>(
   lazyPromises: ((p: T) => Promise<T>)[],
   reducer = (lp1: (p: T) => Promise<T>, lp2: (p: T) => Promise<T>) => (p: T) => lp1(p).then(lp2),
   initialValue?: T
 ) => lazyPromises.reduce(reducer, p => Promise.resolve(p))(initialValue);
 
-export const neverResolves = () => new Promise<never>(() => undefined);
-
-class CustomErrorConstructor<Name extends string> extends CallableInstance<[string], CustomError<Name>> {
-  constructor(name: Name) {
-    super('getError');
-    Object.defineProperty(this.constructor, 'name', {value: name});
-  }
-  getError(message?: string) { return new CustomError(this.constructor.name, message); }
-}
-
-export class CustomError<Name extends string> extends Error {
-  constructor(name: Name, message?: string) {
-    super(message);
-    this.name = name;
-    Object.defineProperty(this.constructor, 'name', {value: name});
-  }
-}
-
-export const customError = <Name extends string>(name: Name) => new CustomErrorConstructor(name);
-
-const chunkArray = <T>(arr: T[], size: number) => {
-  const noOfChunks = Math.ceil(size && arr.length / size);
-  const res = Array<T[]>(noOfChunks);
-  for (let [i, j] = [0, 0]; i < noOfChunks; i++) {
-    res[i] = arr.slice(j, j += size);
-  }
-  return res;
-};
-
-export const rawRegExp = (...args: Parameters<typeof String.raw>) => RegExp(String.raw(...args));
-
-const splitBigNumAt = (num: string, at: number) => num.replace(rawRegExp`^([+-]?)(\d+)(\d{${at}})$`, '$1$2,$1$3')
+const splitBigNumAt = (num: string, at: number) => num.replace(RegExp(String.raw`^([+-]?)(\d+)(\d{${at}})$`), '$1$2,$1$3')
   .replace(/^([^,]*)$/, '0,$1').split(',')
   .map(Number);
 
@@ -86,5 +53,3 @@ export const BigNumOps = {
   lShift: bigNumLShift,
   parse: parseBigNum,
 };
-
-export const Arr = {chunk: chunkArray};

src/wiki.ts

@@ -0,0 +1,203 @@
+import axios from 'axios';
+import fetchCookie = require('fetch-cookie');
+import { writeFileSync } from 'fs';
+import { MWBot } from 'mediawiki2';
+import nodeFetch from 'node-fetch';
+import { firefox } from 'playwright';
+import { CookieJar } from 'tough-cookie';
+
+import { getLogger } from './loggers';
+import { Tweet, processTweetBody } from './twitter';
+
+const logger = getLogger('wiki');
+const baseUrl = 'https://wiki.biligame.com/idolypride';
+
+export default class {
+
+    private bot: MWBot;
+    private lock: ILock;
+
+    constructor(lock: ILock) {
+      this.bot = new MWBot(`${baseUrl}/api.php`);
+      this.lock = lock;
+    }
+    
+    public login = (sessdata: string) =>
+      firefox.launch().then(browser => {
+        const jar = (this.bot as any).cookieJar as CookieJar;
+        return browser.newPage().then(page =>
+          page.context().addCookies([{
+            name: 'SESSDATA',
+            value: sessdata,
+            domain: '.biligame.com',
+            path: '/',
+          }])
+            .then(() => page.route('**/*.{png,jpg,jpeg,gif}', route => route.abort()))
+            .then(() => page.route('*://*.baidu.com/**', route => route.abort()))
+            .then(() => page.goto(`${baseUrl}/index.php?curid=2`, {waitUntil: 'networkidle'}))
+            .then(() => { logger.info('logging in via browser...'); return page.context().cookies(); }) 
+            .then(cookies => {
+              const uidIndex = cookies.findIndex(cookie => cookie.name === 'gamecenter_wiki_UserName');
+              if (!uidIndex) throw new Error('auth error');
+              return Promise.all(cookies.map(({name, value, domain, path}) =>
+                jar.setCookie(`${name}=${value}; Domain=${domain}; Path=${path}`, baseUrl)
+              )).then(() => cookies[uidIndex].value)
+            })
+            .then(uid => {
+              logger.info(`finished logging in via browser, wiki username: ${uid}`);
+              this.bot.fetch = (fetchCookie as any)(nodeFetch, jar);
+              return browser.close();
+            })
+            .catch((err: Error) => browser.close().then(() => {
+              logger.fatal(`error logging in via browser, error: ${err}`);
+              process.exit(0);
+            }))
+        );
+      });
+
+    private fetchMedia = (url: string): Promise<string> => new Promise<ArrayBuffer>((resolve, reject) => {
+      logger.info(`fetching ${url}`);
+      const fetch = () => axios({
+        method: 'get',
+        url,
+        responseType: 'arraybuffer',
+        timeout: 150000,
+      }).then(res => {
+        if (res.status === 200) {
+          logger.info(`successfully fetched ${url}`);
+          resolve(res.data);
+        } else {
+          logger.error(`failed to fetch ${url}: ${res.status}`);
+          reject();
+        }
+      }).catch(err => {
+        logger.error(`failed to fetch ${url}: ${err instanceof Error ? err.message : err}`);
+        logger.info(`trying to fetch ${url} again...`);
+        fetch();
+      });
+      fetch();
+    }).then(data =>
+      (([_, filename, ext]) => {
+        if (ext) {
+          const mediaFileName = `${filename}.${ext}`;
+          writeFileSync(mediaFileName, Buffer.from(data));
+          return mediaFileName;
+        }
+        logger.warn('unable to find MIME type of fetched media, failing this fetch');
+        throw Error();
+      })(/([^\/]*)\?format=([a-z]+)&/.exec(url) ?? /([^\/]*)\.([^?]+)/.exec(url))
+    );
+
+    private uploadMediaItems = (tweet: Tweet, fileNamePrefix: string, indexOffset = 0) => {
+      const mediaItems: Promise<string>[] = [];
+      if (tweet.extended_entities) {
+        tweet.extended_entities.media.forEach((media, index) => {
+          let url;
+          if (media.type === 'photo') {
+            url = media.media_url_https.replace(/\.([a-z]+)$/, '?format=$1') + '&name=orig';
+          } else {
+            url = media.video_info.variants
+              .filter(variant => variant.bitrate !== undefined)
+              .sort((var1, var2) => var2.bitrate - var1.bitrate)
+              .map(variant => variant.url)[0]; // largest video
+          }
+          const mediaPromise = this.fetchMedia(url)
+            .then(mediaFileName => {
+              const filename = `${fileNamePrefix}${indexOffset + index + 1}.${mediaFileName.split('.')[1]}`;
+              logger.info(`uploading ${url} as ${filename}...`);
+              return this.bot.simpleUpload({
+                file: mediaFileName,
+                filename,
+              }).then(() => filename)
+            });
+          mediaItems.push(mediaPromise);
+        });
+      }
+      return Promise.all(mediaItems);
+    };
+
+    public appendMedia = (tweet: Tweet, genre: string, indexOffset: number): Promise<WikiEditResult> => {
+      const {pageTitle} = processTweetBody(tweet);
+      return this.uploadMediaItems(tweet, `公告-${genre}-${pageTitle}-`, indexOffset)
+        .then(fileNames => {
+          logger.info(`updating page 公告/${pageTitle}...`);
+          return this.bot.edit({
+            title: `公告/${pageTitle}`,
+            appendtext: `${fileNames.map(fileName => `[[文件:${fileName}|无框|左]]\n`).join('')}`,
+            bot: true,
+            notminor: true,
+            nocreate: true,
+          })
+            .then(({new: isNewPost, newtimestamp, pageid, result, title}) => ({
+              pageid,
+              title,
+              new: isNewPost,
+              mediafiles: fileNames,
+              result,
+              timestamp: new Date(newtimestamp).toString(),
+            }))
+            .catch(error => {
+              logger.error(`error updating page, error: ${error}`);
+              return {
+                pageid: undefined as number,
+                title: `公告/${pageTitle}`,
+                new: undefined as boolean,
+                mediafiles: [],
+                result: 'Failed',
+                timestamp: undefined as string,
+              };
+            });
+        });
+    };
+
+    public post = (tweet: Tweet, genre: string): Promise<WikiEditResult> => {
+      const {title, body, pageTitle, date} = processTweetBody(tweet);
+      const sameTitleAction = this.lock.lastActions.find(action => action.title === title);
+      if (sameTitleAction) return this.appendMedia(tweet, genre, sameTitleAction.mediafiles.length);
+      return this.uploadMediaItems(tweet, `公告-${genre}-${pageTitle}-`)
+        .then(fileNames => {
+          logger.info(`creating page 公告/${pageTitle}...`);
+          return this.bot.edit({
+            title: `公告/${pageTitle}`,
+            basetimestamp: new Date(),
+            text: `{{文章戳
+|文章上级页面=公告
+|子类别=${genre}
+|时间=${date}
+|作者=IDOLY PRIDE
+|是否原创=否
+|来源=[https://twitter.com/idolypride IDOLY PRIDE]
+|原文地址=[https://twitter.com/idolypride/status/${tweet.id_str} ${pageTitle}]
+}}
+====${title}====
+<poem>
+${body}
+</poem>
+${fileNames.map(fileName => `[[文件:${fileName}|无框|左]]`).join('\n')}
+`,
+            bot: true,
+            notminor: true,
+            createonly: true,
+          })
+            .then(({new: isNewPost, newtimestamp, pageid, result, title}) => ({
+              pageid,
+              title,
+              new: isNewPost,
+              mediafiles: fileNames,
+              result,
+              timestamp: new Date(newtimestamp).toString(),
+            }))
+            .catch(error => {
+              logger.error(`error creating page, error: ${error}`);
+              return {
+                pageid: undefined as number,
+                title: `公告/${pageTitle}`,
+                new: undefined as boolean,
+                mediafiles: [],
+                result: 'Failed',
+                timestamp: undefined as string,
+              };
+            });
+        });
+    };
+}