:construction: render webshot

Signed-off-by: LI JIAHAO <lijiahao99131@gmail.com>

dist/cqhttp.js

@@ -0,0 +1,85 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const CQWebsocket = require("cq-websocket");
+const log4js = require("log4js");
+const helper_1 = require("./helper");
+const logger = log4js.getLogger('cq-websocket');
+logger.level = 'info';
+class default_1 {
+    constructor(opt) {
+        this.retryInterval = 1000;
+        this.initWebsocket = () => {
+            this.bot = new CQWebsocket({
+                access_token: this.botInfo.access_token,
+                enableAPI: true,
+                enableEvent: true,
+                host: this.botInfo.host,
+                port: this.botInfo.port,
+            });
+            this.bot.on('socket.connect', () => {
+                logger.info('websocket connected');
+                this.retryInterval = 1000;
+            });
+            this.bot.on('socket.close', () => {
+                logger.error('websocket closed');
+                this.reconnect();
+            });
+            this.bot.on('socket.error', () => {
+                logger.error('websocket connect error');
+                this.reconnect();
+            });
+            this.bot.on('message', (e, context) => {
+                e.cancel();
+                const chat = {
+                    chatType: context.message_type,
+                    chatID: 0,
+                };
+                switch (context.message_type) {
+                    case "private" /* Private */:
+                        chat.chatID = context.user_id;
+                        break;
+                    case "group" /* Group */:
+                        chat.chatID = context.group_id;
+                        break;
+                    case "discuss" /* Discuss */:
+                        chat.chatID = context.discuss_id;
+                }
+                const cmdObj = helper_1.default(context.raw_message);
+                switch (cmdObj.cmd) {
+                    case 'twitter_sub':
+                    case 'twitter_subscribe':
+                        return this.botInfo.sub(chat, cmdObj.args);
+                    case 'twitter_unsub':
+                    case 'twitter_unsubscribe':
+                        return this.botInfo.unsub(chat, cmdObj.args);
+                    case 'ping':
+                    case 'twitter':
+                        return this.botInfo.list(chat, cmdObj.args);
+                    case 'help':
+                        return `推特搬运机器人：
+/twitter - 查询当前聊天中的订阅
+/twitter_subscribe [链接] - 订阅 Twitter 搬运
+/twitter_unsubscribe [链接] - 退订 Twitter 搬运`;
+                }
+            });
+        };
+        this.connect = () => {
+            this.initWebsocket();
+            logger.warn('connecting to websocket...');
+            this.bot.connect();
+        };
+        this.reconnect = () => {
+            this.retryInterval *= 2;
+            if (this.retryInterval > 300000)
+                this.retryInterval = 300000;
+            logger.info(`retrying in ${this.retryInterval / 1000}s...`);
+            setTimeout(() => {
+                logger.warn('reconnecting to websocket...');
+                this.connect();
+            }, this.retryInterval);
+        };
+        logger.info(`init cqwebsocket for ${opt.host}:${opt.port}, with access_token ${opt.access_token}`);
+        this.botInfo = opt;
+    }
+}
+exports.default = default_1;

dist/main.js

@@ -6,7 +6,7 @@ const fs = require("fs");
 const log4js = require("log4js");
 const path = require("path");
 const command_1 = require("./command");
-const qq_1 = require("./qq");
+const cqhttp_1 = require("./cqhttp");
 const twitter_1 = require("./twitter");
 const logger = log4js.getLogger();
 logger.level = 'info';
@@ -26,7 +26,7 @@ const sections = [
         header: 'Documentation',
         content: [
             'Project home: {underline https://github.com/rikakomoe/cqhttp-twitter-bot}',
-            'Example config: {underline https://qwqq.pw/b96yt}',
+            'Example config: {underline https://qwqq.pw/qpfhg}',
         ],
     },
 ];
@@ -46,6 +46,13 @@ catch (e) {
     console.log(usage);
     process.exit(1);
 }
+if (config.twitter_consumer_key === undefined ||
+    config.twitter_consumer_secret === undefined ||
+    config.twitter_access_token_key === undefined ||
+    config.twitter_access_token_secret === undefined) {
+    console.log('twitter_consumer_key twitter_consumer_secret twitter_access_token_key twitter_access_token_secret are required');
+    process.exit(1);
+}
 if (config.cq_ws_host === undefined) {
     config.cq_ws_host = '127.0.0.1';
     logger.warn('cq_ws_host is undefined, use 127.0.0.1 as default');
@@ -61,6 +68,9 @@ if (config.cq_access_token === undefined) {
 if (config.lockfile === undefined) {
     config.lockfile = 'subscriber.lock';
 }
+if (config.work_interval === undefined) {
+    config.work_interval = 60;
+}
 let lock;
 if (fs.existsSync(path.resolve(config.lockfile))) {
     try {
@@ -95,7 +105,10 @@ else {
         process.exit(1);
     }
 }
-const qq = new qq_1.default({
+Object.keys(lock.threads).forEach(key => {
+    lock.threads[key].offset = -1;
+});
+const qq = new cqhttp_1.default({
     access_token: config.cq_access_token,
     host: config.cq_ws_host,
     port: config.cq_ws_port,
@@ -103,7 +116,14 @@ const qq = new qq_1.default({
     sub: (c, a) => command_1.sub(c, a, lock, config.lockfile),
     unsub: (c, a) => command_1.unsub(c, a, lock, config.lockfile),
 });
-setTimeout(() => {
-    twitter_1.default(lock, config.lockfile);
-}, 60000);
+const worker = new twitter_1.default({
+    consumer_key: config.twitter_consumer_key,
+    consumer_secret: config.twitter_consumer_secret,
+    access_token_key: config.twitter_access_token_key,
+    access_token_secret: config.twitter_access_token_secret,
+    lock,
+    lockfile: config.lockfile,
+    workInterval: config.work_interval,
+});
+setTimeout(worker.work, config.work_interval * 1000);
 qq.connect();

dist/twitter.js

@@ -3,31 +3,77 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fs = require("fs");
 const log4js = require("log4js");
 const path = require("path");
+const Twitter = require("twitter");
 const logger = log4js.getLogger('twitter');
 logger.level = 'info';
-function work(lock, lockfile) {
-    if (lock.feed.length === 0) {
-        setTimeout(() => {
-            work(lock, lockfile);
-        }, 60000);
-        return;
+class default_1 {
+    constructor(opt) {
+        this.work = () => {
+            const lock = this.lock;
+            if (this.workInterval < 1)
+                this.workInterval = 1;
+            if (lock.feed.length === 0) {
+                setTimeout(() => {
+                    this.work();
+                }, this.workInterval * 1000);
+                return;
+            }
+            if (lock.workon >= lock.feed.length)
+                lock.workon = 0;
+            if (!lock.threads[lock.feed[lock.workon]] ||
+                !lock.threads[lock.feed[lock.workon]].subscribers ||
+                lock.threads[lock.feed[lock.workon]].subscribers.length === 0) {
+                logger.error(`nobody subscribes thread ${lock.feed[lock.workon]}, removing from feed`);
+                lock.feed.splice(lock.workon, 1);
+                fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
+                this.work();
+                return;
+            }
+            let match = lock.feed[lock.workon].match(/https:\/\/twitter.com\/([^\/]+)\/lists\/([^\/]+)/);
+            if (match) {
+                const config = {
+                    owner_screen_name: match[1],
+                    slug: match[2],
+                };
+                const offset = lock.threads[lock.feed[lock.workon]].offset;
+                if (offset > 0)
+                    config.since_id = offset;
+                this.client.get('lists/statuses', config, (error, tweets, response) => {
+                    console.log(tweets);
+                });
+            }
+            else {
+                match = lock.feed[lock.workon].match(/https:\/\/twitter.com\/([^\/]+)/);
+                if (match) {
+                    const config = {
+                        screen_name: match[1],
+                    };
+                    const offset = lock.threads[lock.feed[lock.workon]].offset;
+                    if (offset > 0)
+                        config.since_id = offset;
+                    this.client.get('statuses/user_timeline', config, (error, tweets, response) => {
+                        console.log(tweets);
+                    });
+                }
+            }
+            lock.workon++;
+            let timeout = this.workInterval * 1000 / lock.feed.length;
+            if (timeout < 1000)
+                timeout = 1000;
+            fs.writeFileSync(path.resolve(this.lockfile), JSON.stringify(lock));
+            setTimeout(() => {
+                this.work();
+            }, timeout);
+        };
+        this.client = new Twitter({
+            consumer_key: opt.consumer_key,
+            consumer_secret: opt.consumer_secret,
+            access_token_key: opt.access_token_key,
+            access_token_secret: opt.access_token_secret,
+        });
+        this.lockfile = opt.lockfile;
+        this.lock = opt.lock;
+        this.workInterval = opt.workInterval;
     }
-    if (lock.workon >= lock.feed.length)
-        lock.workon = 0;
-    if (!lock.threads[lock.feed[lock.workon]] ||
-        !lock.threads[lock.feed[lock.workon]].subscribers ||
-        lock.threads[lock.feed[lock.workon]].subscribers.length === 0) {
-        logger.error(`nobody subscribes thread ${lock.feed[lock.workon]}, removing from feed`);
-        lock.feed.splice(lock.workon, 1);
-        fs.writeFileSync(path.resolve(lockfile), JSON.stringify(lock));
-        work(lock, lockfile);
-        return;
-    }
-    // TODO: Work on lock.feed[lock.workon]
-    lock.workon++;
-    fs.writeFileSync(path.resolve(lockfile), JSON.stringify(lock));
-    setTimeout(() => {
-        work(lock, lockfile);
-    }, 60000);
 }
-exports.default = work;
+exports.default = default_1;

dist/webshot.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const log4js = require("log4js");
+const webshot = require("webshot");
+const read = require("read-all-stream");
+const logger = log4js.getLogger('webshot');
+logger.level = 'info';
+function renderWebshot(url, height) {
+    let promise = new Promise(resolve => {
+        const options = {
+            windowSize: {
+                width: 1080,
+                height: height,
+            },
+            userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
+            renderDelay: 5000,
+            quality: 100,
+            customCSS: 'html{zoom:2}header{display:none!important}',
+        };
+        const renderStream = webshot(url, options);
+        read(renderStream, 'base64').then(data => {
+            resolve(data);
+        });
+    });
+    return promise.then(data => {
+        if (height < 2048)
+            return renderWebshot(url, height * 2);
+        else
+            return data;
+    });
+}
+/*function fetchImage(): Promise<string> {
+
+}*/
+function default_1(twitter, callback) {
+    twitter.forEach(twi => {
+        const url = `https://mobile.twitter.com/${twi.user.screen_name}/status/${twi.id_str}`;
+        renderWebshot(url, 1920)
+            .then(base64Webshot => {
+            console.log(base64Webshot);
+        });
+    });
+}
+exports.default = default_1;

package.json

@@ -7,11 +7,11 @@
     "lint": "tslint --fix -c tslint.json --project ./"
   },
   "dependencies": {
-    "base64-stream": "^0.1.3",
     "command-line-usage": "^5.0.5",
     "cq-websocket": "^1.2.5",
     "log4js": "^2.10.0",
     "pngjs": "^3.3.3",
+    "read-all-stream": "^3.1.0",
     "twitter": "^1.7.1",
     "typescript": "^2.9.2",
     "webshot": "^0.18.0"

src/webshot.ts

@@ -1,30 +1,43 @@
 import * as log4js from 'log4js';
 import * as webshot from 'webshot';
-import * as base64 from 'base64-stream';
+import * as read from 'read-all-stream';
 
 const logger = log4js.getLogger('webshot');
 logger.level = 'info';
 
-export default function (twitter, callback) {
-  twitter.forEach(twi => {
-    const url = `https://mobile.twitter.com/${twi.user.screen_name}/status/${twi.id_str}`;
+function renderWebshot(url: string, height: number): Promise<string> {
+  let promise = new Promise<string>(resolve => {
     const options = {
       windowSize: {
         width: 1080,
-        height: 1920,
-      },
-      shotSize: {
-        width: 'window',
-        height: 'window',
+        height: height,
       },
       userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
       renderDelay: 5000,
       quality: 100,
       customCSS: 'html{zoom:2}header{display:none!important}',
     };
-    const renderStream = webshot(url, options).pipe(base64.encode());
-    renderStream.on('data', data => {
-      data = 'base64://' + data;
+    const renderStream = webshot(url, options);
+    read(renderStream, 'base64').then(data => {
+      resolve(data);
     });
   });
+  return promise.then(data => {
+    if (height < 2048) return renderWebshot(url, height * 2);
+    else return data;
+  })
+}
+
+/*function fetchImage(): Promise<string> {
+
+}*/
+
+export default function (twitter, callback) {
+  twitter.forEach(twi => {
+    const url = `https://mobile.twitter.com/${twi.user.screen_name}/status/${twi.id_str}`;
+    renderWebshot(url, 1920)
+      .then(base64Webshot => {
+        console.log(base64Webshot);
+      })
+  });
 }

yarn.lock

@@ -117,12 +117,6 @@ balanced-match@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
 
-base64-stream@^0.1.3:
-  version "0.1.3"
-  resolved "https://registry.yarnpkg.com/base64-stream/-/base64-stream-0.1.3.tgz#76b0370b779bb816d12fd41764a6b8573eb5fec3"
-  dependencies:
-    readable-stream "^2.0.2"
-
 bcrypt-pbkdf@^1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e"
@@ -1119,6 +1113,13 @@ raw-body@^2.2.0:
     iconv-lite "0.4.23"
     unpipe "1.0.0"
 
+read-all-stream@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/read-all-stream/-/read-all-stream-3.1.0.tgz#35c3e177f2078ef789ee4bfafa4373074eaef4fa"
+  dependencies:
+    pinkie-promise "^2.0.0"
+    readable-stream "^2.0.0"
+
 readable-stream@1.1.x, "readable-stream@1.x >=1.1.9":
   version "1.1.14"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9"
@@ -1128,7 +1129,7 @@ readable-stream@1.1.x, "readable-stream@1.x >=1.1.9":
     isarray "0.0.1"
     string_decoder "~0.10.x"
 
-readable-stream@2, readable-stream@^2.0.2, readable-stream@^2.2.2, readable-stream@^2.3.0:
+readable-stream@2, readable-stream@^2.0.0, readable-stream@^2.2.2, readable-stream@^2.3.0:
   version "2.3.6"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf"
   dependencies: