|
|
@@ -6,10 +6,9 @@ import {
|
|
|
} from 'instagram-id-to-url-segment';
|
|
|
import {
|
|
|
IgApiClient,
|
|
|
- IgClientError, IgExactUserNotFoundError, IgNetworkError, IgNotFoundError, IgResponseError,
|
|
|
+ IgClientError, IgExactUserNotFoundError, IgResponseError,
|
|
|
MediaInfoResponseItemsItem, UserFeedResponseItemsItem
|
|
|
} from 'instagram-private-api';
|
|
|
-import { RequestError } from 'request-promise/errors';
|
|
|
|
|
|
import { getLogger } from './loggers';
|
|
|
import QQBot, { Message } from './koishi';
|
|
|
@@ -116,7 +115,19 @@ export class ScreenNameNormalizer {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-export let browserLogin = (page: Page): Promise<void> => Promise.reject();
|
|
|
+let browserLogin = (page: Page): Promise<void> => Promise.reject();
|
|
|
+
|
|
|
+let browserSaveCookies = browserLogin;
|
|
|
+
|
|
|
+const acceptCookieConsent = (page: Page) =>
|
|
|
+ page.click('button:has-text("すべて許可")', {timeout: 5000})
|
|
|
+ .then(() => logger.info('accepted cookie consent'))
|
|
|
+ .catch((err: Error) => { if (err.name !== 'TimeoutError') throw err; });
|
|
|
+
|
|
|
+export const WebshotHelpers = {
|
|
|
+ handleLogin: browserLogin,
|
|
|
+ handleCookieConsent: acceptCookieConsent,
|
|
|
+};
|
|
|
|
|
|
export let getPostOwner = (segmentId: string): Promise<string> => Promise.reject();
|
|
|
|
|
|
@@ -126,6 +137,11 @@ export let sendPost = (segmentId: string, receiver: IChat): void => {
|
|
|
|
|
|
export type MediaItem = MediaInfoResponseItemsItem & UserFeedResponseItemsItem;
|
|
|
|
|
|
+export type LazyMediaItem = {
|
|
|
+ pk: string,
|
|
|
+ item: () => Promise<MediaItem>,
|
|
|
+};
|
|
|
+
|
|
|
const logger = getLogger('instagram');
|
|
|
const maxTrials = 3;
|
|
|
const retryInterval = 1500;
|
|
|
@@ -195,24 +211,27 @@ export default class {
|
|
|
logger.warn('cookies will be saved to this file when needed');
|
|
|
}
|
|
|
|
|
|
- browserLogin = (page) => {
|
|
|
- logger.warn('blocked by login dialog, trying to log in manually...');
|
|
|
- return page.type('input[name="username"]', opt.credentials[0])
|
|
|
- .then(() => page.type('input[name="password"]', opt.credentials[1]))
|
|
|
+ browserLogin = page =>
|
|
|
+ page.fill('input[name="username"]', opt.credentials[0])
|
|
|
+ .then(() => logger.warn('blocked by login dialog, trying to log in manually...'))
|
|
|
+ .then(() => page.fill('input[name="password"]', opt.credentials[1]))
|
|
|
.then(() => page.click('button[type="submit"]'))
|
|
|
- .then(() => page.click('button:has-text("情報を保存")'))
|
|
|
- .then(() => page.waitForSelector('img[data-testid="user-avatar"]', {timeout: this.webshotDelay}))
|
|
|
- .then(() => page.context().cookies())
|
|
|
+ .then(() => page.click('button:has-text("情報を保存")'));
|
|
|
+ browserSaveCookies = page =>
|
|
|
+ page.context().cookies()
|
|
|
.then(cookies => {
|
|
|
this.webshotCookies = cookies;
|
|
|
logger.info('successfully logged in, saving cookies to file...');
|
|
|
fs.writeFileSync(path.resolve(this.webshotCookiesLockfile), JSON.stringify(cookies, null, 2), 'utf-8');
|
|
|
- })
|
|
|
+ });
|
|
|
+ WebshotHelpers.handleLogin = page =>
|
|
|
+ browserLogin(page)
|
|
|
+ .then(() => page.waitForSelector('img[data-testid="user-avatar"]', {timeout: this.webshotDelay}))
|
|
|
+ .then(() => browserSaveCookies(page))
|
|
|
.catch((err: Error) => {
|
|
|
if (err.name === 'TimeoutError') logger.warn('navigation timed out, assuming login has failed');
|
|
|
throw err;
|
|
|
});
|
|
|
- };
|
|
|
ScreenNameNormalizer._queryUser = this.queryUser;
|
|
|
const parseMediaError = (err: IgClientError) => {
|
|
|
if (!(err instanceof IgResponseError && err.text === 'Media not found or unavailable')) {
|
|
|
@@ -237,27 +256,108 @@ export default class {
|
|
|
this.wsUrl,
|
|
|
this.mode,
|
|
|
() => this.webshotCookies,
|
|
|
- () => setTimeout(this.work, this.workInterval * 1000)
|
|
|
+ doOnNewPage => {
|
|
|
+ this.queryUserMedia = ((userName, targetId) => {
|
|
|
+ let page: Page;
|
|
|
+ const url = linkBuilder({userName});
|
|
|
+ logger.debug(`pulling ${targetId !== '0' ? `feed ${url} up to ${targetId}` : `top of feed ${url}`}...`);
|
|
|
+ return doOnNewPage(newPage => {
|
|
|
+ page = newPage;
|
|
|
+ let timeout = this.webshotDelay;
|
|
|
+ const startTime = new Date().getTime();
|
|
|
+ const getTimerTime = () => new Date().getTime() - startTime;
|
|
|
+ const getTimeout = () => Math.max(500, timeout - getTimerTime());
|
|
|
+ return page.context().addCookies(this.webshotCookies)
|
|
|
+ .then(() => page.goto(url, {waitUntil: 'load', timeout: getTimeout()}))
|
|
|
+ .then(response => {
|
|
|
+ if (response.status() !== 200) {
|
|
|
+ const err = new Error(
|
|
|
+ `error navigating to user page, error was: ${response.status()} ${response.statusText()}`
|
|
|
+ );
|
|
|
+ throw Object.defineProperty(err, 'name', {
|
|
|
+ value: 'ResponseError',
|
|
|
+ });
|
|
|
+ }
|
|
|
+ }).then(() => acceptCookieConsent(page))
|
|
|
+ .then(() =>
|
|
|
+ (next => Promise.race([
|
|
|
+ browserLogin(page)
|
|
|
+ .catch((err: Error) => {
|
|
|
+ if (err.name === 'TimeoutError') logger.warn('navigation timed out, assuming login has failed');
|
|
|
+ throw err;
|
|
|
+ })
|
|
|
+ .then(() => browserSaveCookies(page))
|
|
|
+ .then(() => page.goto(url)).then(next),
|
|
|
+ next(),
|
|
|
+ ]))(() => page.waitForSelector('article', {timeout: getTimeout()}))
|
|
|
+ ).then(handle => {
|
|
|
+ const postHandler = () => {
|
|
|
+ const toId = (href: string) => urlSegmentToId((/\/p\/(.*)\/$/.exec(href) ?? [])[1]);
|
|
|
+ if (targetId === '0') {
|
|
|
+ return handle.$$eval('a', as =>
|
|
|
+ as.filter(a => !a.querySelector('[aria-label="IGTV"]'))[0].href
|
|
|
+ ).then(href => href ? [toId(href)] : null);
|
|
|
+ }
|
|
|
+ return handle.$$eval('a', as =>
|
|
|
+ as.filter(a => !a.querySelector('[aria-label="IGTV"]')).map(a => a.href)
|
|
|
+ ).then(hrefs => {
|
|
|
+ let id: string;
|
|
|
+ const itemIds: string[] = [];
|
|
|
+ for (const href of hrefs) {
|
|
|
+ id = toId(href);
|
|
|
+ if (id && BigNumOps.compare(id, targetId) > 0) itemIds.push(id);
|
|
|
+ else return itemIds;
|
|
|
+ }
|
|
|
+ return null; // has more
|
|
|
+ });
|
|
|
+ };
|
|
|
+ return postHandler().then(itemIds => {
|
|
|
+ if (itemIds) return itemIds;
|
|
|
+ timeout += this.webshotDelay / 2;
|
|
|
+ return handle.$$('a')
|
|
|
+ .then(as => { as.pop().scrollIntoViewIfNeeded(); return as.length + 1; })
|
|
|
+ .then(loadedCount => page.waitForFunction(count =>
|
|
|
+ document.querySelectorAll('article a').length > count
|
|
|
+ , loadedCount))
|
|
|
+ .then(postHandler);
|
|
|
+ });
|
|
|
+ }).catch((err: Error) => {
|
|
|
+ if (err.name !== 'TimeoutError' && err.name !== 'ResponseError') throw err;
|
|
|
+ if (err.name === 'ResponseError') {
|
|
|
+ logger.warn(`error while fetching tweets for ${userName}: ${err.message}`);
|
|
|
+ } else logger.warn(`navigation timed out at ${getTimerTime()} ms`);
|
|
|
+ return [] as string[];
|
|
|
+ }).then(itemIds => itemIds.map(id => this.lazyGetMediaById(id)));
|
|
|
+ }).finally(() => { page.close(); });
|
|
|
+ });
|
|
|
+ setTimeout(this.work, this.workInterval * 1000);
|
|
|
+ }
|
|
|
);
|
|
|
};
|
|
|
|
|
|
+ public queryUserMedia: (username: string, targetId?: string) => Promise<LazyMediaItem[]>;
|
|
|
+
|
|
|
public queryUser = (username: string) => this.client.user.searchExact(username)
|
|
|
.then(user => `${user.username}:${user.pk}`);
|
|
|
|
|
|
private workOnMedia = (
|
|
|
- mediaItems: MediaItem[],
|
|
|
+ lazyMediaItems: LazyMediaItem[],
|
|
|
sendMedia: (msg: string, text: string, author: string) => void
|
|
|
- ) => this.webshot(mediaItems, sendMedia, this.webshotDelay);
|
|
|
+ ) => this.webshot(lazyMediaItems, sendMedia, this.webshotDelay);
|
|
|
|
|
|
public urlSegmentToId = urlSegmentToId;
|
|
|
|
|
|
- public getMedia = (segmentId: string, sender: (msg: string, text: string, author: string) => void) =>
|
|
|
- this.client.media.info(urlSegmentToId(segmentId))
|
|
|
- .then(media => {
|
|
|
- const mediaItem = media.items[0] as MediaItem;
|
|
|
- logger.debug(`api returned media post ${JSON.stringify(mediaItem)} for query id=${segmentId}`);
|
|
|
- return this.workOnMedia([mediaItem], sender);
|
|
|
- });
|
|
|
+ public lazyGetMediaById = (id: string): LazyMediaItem => ({
|
|
|
+ pk: id,
|
|
|
+ item: () => this.client.media.info(id).then(media => {
|
|
|
+ const mediaItem = media.items[0] as MediaItem;
|
|
|
+ logger.debug(`api returned media post ${JSON.stringify(mediaItem)} for query id=${id}`);
|
|
|
+ return mediaItem;
|
|
|
+ }),
|
|
|
+ });
|
|
|
+
|
|
|
+ private getMedia = (segmentId: string, sender: (msg: string, text: string, author: string) => void) =>
|
|
|
+ this.workOnMedia([this.lazyGetMediaById(urlSegmentToId(segmentId))], sender);
|
|
|
|
|
|
private sendMedia = (source?: string, ...to: IChat[]) => (msg: string, text: string, author: string) => {
|
|
|
to.forEach(subscriber => {
|
|
|
@@ -298,39 +398,16 @@ export default class {
|
|
|
}
|
|
|
|
|
|
const currentFeed = lock.feed[lock.workon];
|
|
|
- logger.debug(`pulling feed ${currentFeed}`);
|
|
|
|
|
|
- const promise = new Promise<UserFeedResponseItemsItem[]>(resolve => {
|
|
|
+ const promise = new Promise<LazyMediaItem[]>(resolve => {
|
|
|
const match = /https:\/\/www\.instagram\.com\/([^\/]+)/.exec(currentFeed);
|
|
|
if (match) {
|
|
|
- const feed = this.client.feed.user(lock.threads[currentFeed].id);
|
|
|
- const newer = (item: UserFeedResponseItemsItem) =>
|
|
|
- BigNumOps.compare(item.pk, lock.threads[currentFeed].offset) > 0;
|
|
|
- const fetchMore = () => new Promise<UserFeedResponseItemsItem[]>(fetch => {
|
|
|
- feed.request().then(response => {
|
|
|
- if (response.items.length === 0) return fetch([]);
|
|
|
- if (response.items.every(newer)) {
|
|
|
- fetchMore().then(fetched => fetch(response.items.concat(fetched)));
|
|
|
- } else fetch(response.items.filter(newer));
|
|
|
- }, (error: IgClientError & Partial<RequestError>) => {
|
|
|
- if (error instanceof IgNetworkError) {
|
|
|
- logger.warn(`error on fetching media for ${currentFeed}: ${JSON.stringify(error.cause)}`);
|
|
|
- if (!(error instanceof IgNotFoundError)) return;
|
|
|
- lock.threads[currentFeed].subscribers.forEach(subscriber => {
|
|
|
- logger.info(`sending notfound message of ${currentFeed} to ${JSON.stringify(subscriber)}`);
|
|
|
- this.bot.sendTo(subscriber, `链接 ${currentFeed} 指向的用户或列表不存在，请退订。`).catch();
|
|
|
- });
|
|
|
- } else {
|
|
|
- logger.error(`unhandled error on fetching media for ${currentFeed}: ${JSON.stringify(error)}`);
|
|
|
- }
|
|
|
- fetch([]);
|
|
|
- });
|
|
|
- });
|
|
|
- fetchMore().then(resolve);
|
|
|
+ resolve(this.queryUserMedia(match[1], this.lock.threads[currentFeed].offset));
|
|
|
}
|
|
|
+ resolve([]);
|
|
|
});
|
|
|
|
|
|
- promise.then((mediaItems: MediaItem[]) => {
|
|
|
+ promise.then((mediaItems: LazyMediaItem[]) => {
|
|
|
const currentThread = lock.threads[currentFeed];
|
|
|
|
|
|
const updateDate = () => currentThread.updatedAt = new Date().toString();
|
