:sparkles: remove duplicate tweets

README.md

@@ -44,6 +44,10 @@ cqhttp-twitter-bot config.json
 | webshot_delay | 抓取网页截图时等待网页加载的延迟时长（毫秒） | 5000 |
 | lockfile | 本地保存订阅信息以便下次启动时恢复 | subscriber.lock |
 | loglevel | 日志调试等级 | info |
+| redis | 启用 Redis | false |
+| redis_host | Redis Host | 127.0.0.1 |
+| redis_port | Redis Port | 6379 |
+| redis_expire_time | Redis key 过期时间（秒） | 43200 |
 
 示例文件在 `config.example.json`
 
@@ -82,15 +86,10 @@ Bot 启动了以后就可以在 QQ 里用命令了。命令有：
 3. webshot_delay 如果设成 0 的话肯定不行的，会出现正在加载的界面。这个具体多
 少最合适可以自己试，5 秒应该是比较保险了。
 
-4. 如果在同一个聊天里的会话有重复，不会被去重。例如在某聊天中同时订阅了特朗普和
-包含特朗普的列表 A，那么每次特朗普发推你都会收到两条一样的推送。这个是因为 API 的构造
-本身决定的，要是给你去重的话还得存给你推过哪些，很麻烦，懒得做，意义不大，就注意不要
-这样就好了。
+4. 如果启用了 Redis，同一聊天中 Redis 过期时间内的重复推文会被去重，
+如同一列表中不同人转推等。
 
-5. 列表中是没有回复的。实际上你看 Twitter 的列表本来也没有回复。个人的时间轴会显示
-回复。
-
-6. 转推是没有图片的。这个也是 API 的问题。这个人原创的 twitter 的话如果有图片
+5. 转推是没有图片的。这个是 API 的问题。这个人原创的 twitter 的话如果有图片
 会另外拉取一波图片附加在消息里。如果是视频的话则是封面图。
 
-7. 怎么查看翻译？QQ 点开图片，长按 -> 提取图中文字 -> 译
+6. 怎么查看翻译？QQ 点开图片，长按 -> 提取图中文字 -> 译

config.example.json

@@ -9,5 +9,9 @@
   "work_interval": 60,
   "webshot_delay": 5000,
   "lockfile": "subscriber.lock",
-  "loglevel": "info"
+  "loglevel": "info",
+  "redis": true,
+  "redis_host": "127.0.0.1",
+  "redis_port": 6379,
+  "redis_expire_time": 43200
 }

dist/main.js

@@ -74,6 +74,14 @@ if (config.webshot_delay === undefined) {
 if (config.loglevel === undefined) {
     config.loglevel = 'info';
 }
+let redisConfig;
+if (config.redis) {
+    redisConfig = {
+        redisHost: config.redis_host || "127.0.0.1",
+        redisPort: config.redis_port || 6379,
+        redisExpireTime: config.redis_expire_time || 43200,
+    };
+}
 global.loglevel = config.loglevel;
 const command_1 = require("./command");
 const cqhttp_1 = require("./cqhttp");
@@ -133,6 +141,7 @@ const worker = new twitter_1.default({
     workInterval: config.work_interval,
     bot: qq,
     webshotDelay: config.webshot_delay,
+    redis: redisConfig,
 });
 worker.launch();
 qq.connect();

dist/twitter.js

@@ -3,6 +3,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const fs = require("fs");
 const log4js = require("log4js");
 const path = require("path");
+const redis = require("redis");
+const sha1 = require("sha1");
 const Twitter = require("twitter");
 const webshot_1 = require("./webshot");
 const logger = log4js.getLogger('twitter');
@@ -10,6 +12,12 @@ logger.level = global.loglevel;
 class default_1 {
     constructor(opt) {
         this.launch = () => {
+            if (this.redisConfig) {
+                this.redisClient = redis.createClient({
+                    host: this.redisConfig.redisHost,
+                    port: this.redisConfig.redisPort,
+                });
+            }
             this.webshot = new webshot_1.default(() => setTimeout(this.work, this.workInterval * 1000));
         };
         this.work = () => {
@@ -99,9 +107,14 @@ class default_1 {
                 }
                 if (lock.threads[lock.feed[lock.workon]].offset === 0)
                     tweets.splice(1);
-                return this.webshot(tweets, msg => {
+                return this.webshot(tweets, (msg, text) => {
                     lock.threads[lock.feed[lock.workon]].subscribers.forEach(subscriber => {
                         logger.info(`pushing data of thread ${lock.feed[lock.workon]} to ${JSON.stringify(subscriber)}`);
+                        const hash = sha1(JSON.stringify(subscriber) + text);
+                        if (this.redisClient && this.redisClient.exists(hash)) {
+                            logger.info('key hash exists, skip this subscriber');
+                            return;
+                        }
                         this.bot.bot('send_msg', {
                             message_type: subscriber.chatType,
                             user_id: subscriber.chatID,
@@ -109,6 +122,10 @@ class default_1 {
                             discuss_id: subscriber.chatID,
                             message: msg,
                         });
+                        if (this.redisClient) {
+                            this.redisClient.set(hash, 'true');
+                            this.redisClient.expire(hash, this.redisConfig.redisExpireTime);
+                        }
                     });
                 }, this.webshotDelay)
                     .then(() => {
@@ -138,6 +155,7 @@ class default_1 {
         this.workInterval = opt.workInterval;
         this.bot = opt.bot;
         this.webshotDelay = opt.webshotDelay;
+        this.redisConfig = opt.redis;
     }
 }
 exports.default = default_1;

dist/webshot.js

@@ -167,7 +167,7 @@ class Webshot extends CallableInstance {
                     }
                 });
             }
-            promise.then(() => callback(cqstr));
+            promise.then(() => callback(cqstr, twi.full_text));
         });
         return promise;
     }

package.json

@@ -35,6 +35,8 @@
     "pngjs": "^3.3.3",
     "puppeteer": "^1.5.0",
     "read-all-stream": "^3.1.0",
+    "redis": "^2.8.0",
+    "sha1": "^1.1.1",
     "twitter": "^1.7.1",
     "typescript": "^2.9.2"
   },
@@ -42,6 +44,7 @@
     "@types/node": "^10.5.1",
     "@types/pngjs": "^3.3.2",
     "@types/puppeteer": "^1.5.0",
+    "@types/redis": "^2.8.6",
     "tslint": "^5.10.0",
     "tslint-config-prettier": "^1.13.0"
   }

src/main.ts

@@ -80,6 +80,14 @@ if (config.webshot_delay === undefined) {
 if (config.loglevel === undefined) {
   config.loglevel = 'info';
 }
+let redisConfig: IRedisConfig;
+if (config.redis) {
+  redisConfig = {
+    redisHost: config.redis_host || "127.0.0.1",
+    redisPort: config.redis_port || 6379,
+    redisExpireTime: config.redis_expire_time || 43200,
+  };
+}
 
 (global as any).loglevel = config.loglevel;
 
@@ -142,6 +150,7 @@ const worker = new Worker({
   workInterval: config.work_interval,
   bot: qq,
   webshotDelay: config.webshot_delay,
+  redis: redisConfig,
 });
 worker.launch();
 

src/model.d.ts

@@ -20,4 +20,10 @@ interface ILock {
         subscribers: IChat[],
       }
   }
-}
+}
+
+interface IRedisConfig {
+  redisHost: string,
+  redisPort: number,
+  redisExpireTime: number
+}

src/twitter.ts

@@ -1,6 +1,9 @@
 import * as fs from 'fs';
 import * as log4js from 'log4js';
 import * as path from 'path';
+import * as redis from 'redis';
+import { RedisClient } from 'redis';
+import * as sha1 from 'sha1';
 import * as Twitter from 'twitter';
 
 import QQBot from './cqhttp';
@@ -16,6 +19,7 @@ interface IWorkerOption {
   consumer_secret: string;
   access_token_key: string;
   access_token_secret: string;
+  redis: IRedisConfig;
 }
 
 const logger = log4js.getLogger('twitter');
@@ -30,6 +34,8 @@ export default class {
   private bot: QQBot;
   private webshotDelay: number;
   private webshot: Webshot;
+  private redisConfig: IRedisConfig;
+  private redisClient: RedisClient;
 
   constructor(opt: IWorkerOption) {
     this.client = new Twitter({
@@ -43,9 +49,16 @@ export default class {
     this.workInterval = opt.workInterval;
     this.bot = opt.bot;
     this.webshotDelay = opt.webshotDelay;
+    this.redisConfig = opt.redis;
   }
 
   public launch = () => {
+    if (this.redisConfig) {
+      this.redisClient = redis.createClient({
+        host: this.redisConfig.redisHost,
+        port: this.redisConfig.redisPort,
+      });
+    }
     this.webshot = new Webshot(() => setTimeout(this.work, this.workInterval * 1000));
   }
 
@@ -132,9 +145,14 @@ export default class {
         return;
       }
       if (lock.threads[lock.feed[lock.workon]].offset === 0) tweets.splice(1);
-      return (this.webshot as any)(tweets, msg => {
+      return (this.webshot as any)(tweets, (msg, text) => {
         lock.threads[lock.feed[lock.workon]].subscribers.forEach(subscriber => {
           logger.info(`pushing data of thread ${lock.feed[lock.workon]} to ${JSON.stringify(subscriber)}`);
+          const hash = sha1(JSON.stringify(subscriber) + text);
+          if (this.redisClient && this.redisClient.exists(hash)) {
+            logger.info('key hash exists, skip this subscriber');
+            return;
+          }
           this.bot.bot('send_msg', {
             message_type: subscriber.chatType,
             user_id: subscriber.chatID,
@@ -142,6 +160,10 @@ export default class {
             discuss_id: subscriber.chatID,
             message: msg,
           });
+          if (this.redisClient) {
+            this.redisClient.set(hash, 'true');
+            this.redisClient.expire(hash, this.redisConfig.redisExpireTime);
+          }
         });
       }, this.webshotDelay)
         .then(() => {

src/webshot.ts

@@ -168,7 +168,7 @@ class Webshot extends CallableInstance {
           }
         });
       }
-      promise.then(() => callback(cqstr));
+      promise.then(() => callback(cqstr, twi.full_text));
     });
     return promise;
   }

yarn.lock

@@ -23,6 +23,13 @@
     "@types/events" "*"
     "@types/node" "*"
 
+"@types/redis@^2.8.6":
+  version "2.8.6"
+  resolved "https://registry.yarnpkg.com/@types/redis/-/redis-2.8.6.tgz#3674d07a13ad76bccda4c37dc3909e4e95757e7e"
+  dependencies:
+    "@types/events" "*"
+    "@types/node" "*"
+
 addressparser@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/addressparser/-/addressparser-1.0.1.tgz#47afbe1a2a9262191db6838e4fd1d39b40821746"
@@ -231,6 +238,10 @@ chalk@^2.3.0, chalk@^2.4.1:
     escape-string-regexp "^1.0.5"
     supports-color "^5.3.0"
 
+"charenc@>= 0.0.1":
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667"
+
 circular-json@^0.5.4:
   version "0.5.4"
   resolved "https://registry.yarnpkg.com/circular-json/-/circular-json-0.5.4.tgz#ff1ad2f2e392eeb8a5172d4d985fa846ed8ad656"
@@ -292,6 +303,10 @@ cq-websocket@^1.2.6:
     lodash.get "^4.4.2"
     websocket "^1.0.25"
 
+"crypt@>= 0.0.1":
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b"
+
 cryptiles@2.x.x:
   version "2.0.5"
   resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-2.0.5.tgz#3bdfecdc608147c1c67202fa291e7dca59eaa3b8"
@@ -1125,7 +1140,7 @@ redis-parser@^2.6.0:
   version "2.6.0"
   resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-2.6.0.tgz#52ed09dacac108f1a631c07e9b69941e7a19504b"
 
-redis@^2.7.1:
+redis@^2.7.1, redis@^2.8.0:
   version "2.8.0"
   resolved "https://registry.yarnpkg.com/redis/-/redis-2.8.0.tgz#202288e3f58c49f6079d97af7a10e1303ae14b02"
   dependencies:
@@ -1225,6 +1240,13 @@ setprototypeof@1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.0.tgz#d0bd85536887b6fe7c0d818cb962d9d91c54e656"
 
+sha1@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/sha1/-/sha1-1.1.1.tgz#addaa7a93168f393f19eb2b15091618e2700f848"
+  dependencies:
+    charenc ">= 0.0.1"
+    crypt ">= 0.0.1"
+
 slack-node@~0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/slack-node/-/slack-node-0.2.0.tgz#de4b8dddaa8b793f61dbd2938104fdabf37dfa30"