Merge branch 'koishi-production' into mediaonly-koishi

dist/twitter.js

@@ -245,11 +245,12 @@ class default_1 {
                 }
                 if (endpoint) {
                     const offset = lock.threads[currentFeed].offset;
+                    config.include_rts = false;
                     if (offset > 0)
                         config.since_id = offset;
                     if (offset < -1)
                         config.max_id = offset.slice(1);
-                    this.client.get(endpoint, config, (error, tweets, response) => {
+                    const getMore = (gotTweets = []) => this.client.get(endpoint, config, (error, tweets) => {
                         if (error) {
                             if (error instanceof Array && error.length > 0 && error[0].code === 34) {
                                 logger.warn(`error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
@@ -261,11 +262,13 @@ class default_1 {
                             else {
                                 logger.error(`unhandled error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
                             }
-                            resolve([]);
                         }
-                        else
-                            resolve(tweets);
+                        if (!tweets || tweets.length <= 1)
+                            return resolve(gotTweets);
+                        config.max_id = tweets.slice(-1)[0].id_str;
+                        getMore(gotTweets.concat(tweets));
                     });
+                    getMore();
                 }
             });
             promise.then((tweets) => {
@@ -284,7 +287,7 @@ class default_1 {
                 logger.info(`current offset: ${currentThread.offset}, current top of feed: ${topOfFeed}`);
                 const bottomOfFeed = tweets[tweets.length - 1].id_str;
                 const updateOffset = () => setOffset(topOfFeed);
-                tweets = tweets.filter(twi => !twi.retweeted_status && twi.extended_entities);
+                tweets = tweets.filter(twi => twi.extended_entities);
                 logger.info(`found ${tweets.length} tweets with extended entities`);
                 if (currentThread.offset === '-1') {
                     updateOffset();

src/twitter.ts

@@ -407,10 +407,13 @@ export default class {
       }
 
       if (endpoint) {
-        const offset = lock.threads[currentFeed].offset as unknown as number;
-        if (offset > 0) config.since_id = offset;
-        if (offset < -1) config.max_id = (offset as unknown as string).slice(1) as unknown as number;
-        this.client.get(endpoint, config, (error: {[key: string]: any}[], tweets, response) => {
+        const offset = lock.threads[currentFeed].offset;
+        config.include_rts = false;
+        if (offset as unknown as number > 0) config.since_id = offset;
+        if (offset as unknown as number < -1) config.max_id = offset.slice(1);
+        const getMore = (gotTweets: Tweets = []) => this.client.get(
+          endpoint, config, (error: {[key: string]: any}[], tweets: Tweets
+        ) => {
           if (error) {
             if (error instanceof Array && error.length > 0 && error[0].code === 34) {
               logger.warn(`error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
@@ -421,9 +424,12 @@ export default class {
             } else {
               logger.error(`unhandled error on fetching tweets for ${currentFeed}: ${JSON.stringify(error)}`);
             }
-            resolve([]);
-          } else resolve(tweets);
+          }
+          if (!tweets || tweets.length <= 1) return resolve(gotTweets);
+          config.max_id = tweets.slice(-1)[0].id_str;
+          getMore(gotTweets.concat(tweets));
         });
+        getMore();
       }
     });
 
@@ -445,7 +451,7 @@ export default class {
       logger.info(`current offset: ${currentThread.offset}, current top of feed: ${topOfFeed}`);
       const bottomOfFeed = tweets[tweets.length - 1].id_str;
       const updateOffset = () => setOffset(topOfFeed);
-      tweets = tweets.filter(twi => !twi.retweeted_status && twi.extended_entities);
+      tweets = tweets.filter(twi => twi.extended_entities);
       logger.info(`found ${tweets.length} tweets with extended entities`);
       if (currentThread.offset === '-1') { updateOffset(); return; }
       if (currentThread.offset as unknown as number <= 0) {