add pagination delay, add comments

dist/twitter.js

@@ -202,9 +202,10 @@ class default_1 {
                     logger.debug(`pulling ${targetId !== '0' ? `feed ${url} up to ${targetId}` : `top of feed ${url}`}...`);
                     return doOnNewPage(newPage => {
                         page = newPage;
+                        let timeout = this.webshotDelay / 2;
                         const startTime = new Date().getTime();
                         const getTimerTime = () => new Date().getTime() - startTime;
-                        const getTimeout = () => isWaitingForLogin ? 0 : Math.max(5000, this.webshotDelay / 2 - getTimerTime());
+                        const getTimeout = () => isWaitingForLogin ? 0 : Math.max(5000, timeout - getTimerTime());
                         return page.context().addCookies(this.webshotCookies)
                             .then(() => page.goto(url, { waitUntil: 'load', timeout: getTimeout() }))
                             .then(response => {
@@ -256,7 +257,10 @@ class default_1 {
                                     return itemIds;
                                 logger.info('unable to find a smaller id than target, trying on next page...');
                                 url = graphqlLinkBuilder({ userId: user.id, after: pageInfo.end_cursor });
-                                return page.goto(url, { waitUntil: 'load', timeout: getTimeout() })
+                                const nextPageDelay = this.webshotDelay * (0.4 + Math.random() * 0.1);
+                                timeout += nextPageDelay;
+                                return util_1.promisify(setTimeout)(nextPageDelay)
+                                    .then(() => page.goto(url, { waitUntil: 'load', timeout: getTimeout() }))
                                     .then(responseHandler)
                                     .then(({ data }) => jsonHandler(data));
                             };

src/twitter.ts

@@ -389,9 +389,10 @@ export default class {
           logger.debug(`pulling ${targetId !== '0' ? `feed ${url} up to ${targetId}` : `top of feed ${url}`}...`);
           return doOnNewPage(newPage => {
             page = newPage;
+            let timeout = this.webshotDelay / 2;
             const startTime = new Date().getTime();
             const getTimerTime = () => new Date().getTime() - startTime;
-            const getTimeout = () => isWaitingForLogin ? 0 : Math.max(5000, this.webshotDelay / 2 - getTimerTime());
+            const getTimeout = () => isWaitingForLogin ? 0 : Math.max(5000, timeout - getTimerTime());
             return page.context().addCookies(this.webshotCookies)
               .then(() => page.goto(url, {waitUntil: 'load', timeout: getTimeout()}))
               .then(response => {
@@ -432,15 +433,24 @@ export default class {
                 const jsonHandler = ({user}: {user: IgGraphQLUser}): string[] | Promise<string[]> => {
                   const pageInfo = user.edge_owner_to_timeline_media.page_info;
                   for (const {node} of user.edge_owner_to_timeline_media.edges) {
+                    // exclude IGTV
                     if (node.__typename === 'GraphVideo' && node.product_type === 'igtv') continue;
+                    // add post if ID is greater than target
                     if (node.id && BigNumOps.compare(node.id, targetId) > 0) itemIds.push(node.id);
+                    // return of ID is equal to or smaller than target
                     else return itemIds;
+                    // return after first addition if newly subscribed or restarted with resuming disabled
                     if (Number(targetId) < 1) return itemIds;
                   }
+                  // return if all IDs are greater than target but end of feed is reached
                   if (!pageInfo?.has_next_page) return itemIds;
+                  // else, fetch next page using end_cursor
                   logger.info('unable to find a smaller id than target, trying on next page...');
                   url = graphqlLinkBuilder({userId: user.id, after: pageInfo.end_cursor});
-                  return page.goto(url, {waitUntil: 'load', timeout: getTimeout()})
+                  const nextPageDelay = this.webshotDelay * (0.4 + Math.random() * 0.1);
+                  timeout += nextPageDelay;
+                  return promisify(setTimeout)(nextPageDelay)
+                    .then(() => page.goto(url, {waitUntil: 'load', timeout: getTimeout()}))
                     .then(responseHandler)
                     .then(({data}: {data: {user: IgGraphQLUser}}) => jsonHandler(data));
                 };