|
|
@@ -1,25 +1,22 @@
|
|
|
import * as fs from 'fs';
|
|
|
import * as path from 'path';
|
|
|
-import {
|
|
|
- instagramIdToUrlSegment as idToUrlSegment,
|
|
|
- urlSegmentToInstagramId as urlSegmentToId
|
|
|
-} from 'instagram-id-to-url-segment';
|
|
|
import {
|
|
|
IgApiClient,
|
|
|
- IgClientError, IgExactUserNotFoundError, IgNetworkError, IgNotFoundError, IgResponseError,
|
|
|
- MediaInfoResponseItemsItem, UserFeedResponseItemsItem
|
|
|
+ IgClientError, IgExactUserNotFoundError,
|
|
|
+ IgNetworkError,
|
|
|
+ ReelsMediaFeedResponseItem, UserFeedResponseUser
|
|
|
} from 'instagram-private-api';
|
|
|
import { RequestError } from 'request-promise/errors';
|
|
|
|
|
|
import { getLogger } from './loggers';
|
|
|
import QQBot, { Message } from './koishi';
|
|
|
import { BigNumOps } from './utils';
|
|
|
-import Webshot, { Cookies, Page } from './webshot';
|
|
|
+import Webshot from './webshot';
|
|
|
|
|
|
-const parseLink = (link: string): {userName?: string, postUrlSegment?: string} => {
|
|
|
+const parseLink = (link: string): {userName?: string, storyId?: string} => {
|
|
|
let match =
|
|
|
- /instagram\.com\/p\/([A-Za-z0-9\-_]+)/.exec(link);
|
|
|
- if (match) return {postUrlSegment: match[1]};
|
|
|
+ /instagram\.com\/stories\/([^\/?#]+)\/(\d+)/.exec(link);
|
|
|
+ if (match) return {userName: ScreenNameNormalizer.normalize(match[1]).split(':')[0], storyId: match[2]};
|
|
|
match =
|
|
|
/instagram\.com\/([^\/?#]+)/.exec(link) ||
|
|
|
/^([^\/?#]+)$/.exec(link);
|
|
|
@@ -27,14 +24,13 @@ const parseLink = (link: string): {userName?: string, postUrlSegment?: string} =
|
|
|
return;
|
|
|
};
|
|
|
|
|
|
-const isValidUrlSegment = (input: string) => /^[A-Za-z0-9\-_]+$/.test(input);
|
|
|
-
|
|
|
const linkBuilder = (config: ReturnType<typeof parseLink>): string => {
|
|
|
- if (config.userName) return `https://www.instagram.com/${config.userName}/`;
|
|
|
- if (config.postUrlSegment) return `https://www.instagram.com/p/${config.postUrlSegment}/`;
|
|
|
+ if (!config.userName) return;
|
|
|
+ if (!config.storyId) return `https://www.instagram.com/${config.userName}/`;
|
|
|
+ return `https://www.instagram.com/stories/${config.userName}/${config.storyId}/`;
|
|
|
};
|
|
|
|
|
|
-export {linkBuilder, parseLink, isValidUrlSegment, idToUrlSegment, urlSegmentToId};
|
|
|
+export {linkBuilder, parseLink};
|
|
|
|
|
|
interface IWorkerOption {
|
|
|
sessionLockfile: string;
|
|
|
@@ -116,15 +112,11 @@ export class ScreenNameNormalizer {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-export let browserLogin = (page: Page): Promise<void> => Promise.reject();
|
|
|
-
|
|
|
-export let getPostOwner = (segmentId: string): Promise<string> => Promise.reject();
|
|
|
-
|
|
|
-export let sendPost = (segmentId: string, receiver: IChat): void => {
|
|
|
+export let sendAllStories = (segmentId: string, receiver: IChat): void => {
|
|
|
throw Error();
|
|
|
};
|
|
|
|
|
|
-export type MediaItem = MediaInfoResponseItemsItem & UserFeedResponseItemsItem;
|
|
|
+export type MediaItem = ReelsMediaFeedResponseItem;
|
|
|
|
|
|
const logger = getLogger('instagram');
|
|
|
const maxTrials = 3;
|
|
|
@@ -163,8 +155,6 @@ export default class {
|
|
|
private workInterval: number;
|
|
|
private bot: QQBot;
|
|
|
private webshotDelay: number;
|
|
|
- private webshotCookies: Cookies;
|
|
|
- private webshotCookiesLockfile: string;
|
|
|
private webshot: Webshot;
|
|
|
private mode: number;
|
|
|
private wsUrl: string;
|
|
|
@@ -175,7 +165,6 @@ export default class {
|
|
|
this.client = new IgApiClient();
|
|
|
this.session = new SessionManager(this.client, opt.sessionLockfile, opt.credentials);
|
|
|
this.lockfile = opt.lockfile;
|
|
|
- this.webshotCookiesLockfile = opt.webshotCookiesLockfile;
|
|
|
this.lock = opt.lock;
|
|
|
this.workInterval = opt.workInterval;
|
|
|
this.bot = opt.bot;
|
|
|
@@ -183,82 +172,75 @@ export default class {
|
|
|
this.mode = opt.mode;
|
|
|
this.wsUrl = opt.wsUrl;
|
|
|
|
|
|
- const cookiesFilePath = path.resolve(this.webshotCookiesLockfile);
|
|
|
- if (fs.existsSync(cookiesFilePath)) {
|
|
|
- try {
|
|
|
- this.webshotCookies = JSON.parse(fs.readFileSync(cookiesFilePath, 'utf8')) as Cookies;
|
|
|
- logger.info(`loaded webshot cookies from file ${this.webshotCookiesLockfile}`);
|
|
|
- } catch (err) {
|
|
|
- logger.warn(`failed to load webshot cookies from file ${this.webshotCookiesLockfile}: `, err);
|
|
|
- logger.warn('cookies will be saved to this file when needed');
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- browserLogin = (page) => {
|
|
|
- logger.warn('blocked by login dialog, trying to log in manually...');
|
|
|
- return page.type('input[name="username"]', opt.credentials[0])
|
|
|
- .then(() => page.type('input[name="password"]', opt.credentials[1]))
|
|
|
- .then(() => page.click('button[type="submit"]'))
|
|
|
- .then(() => page.click('button:has-text("情報を保存")'))
|
|
|
- .then(() => page.waitForSelector('img[data-testid="user-avatar"]', {timeout: this.webshotDelay}))
|
|
|
- .then(() => page.context().cookies())
|
|
|
- .then(cookies => {
|
|
|
- this.webshotCookies = cookies;
|
|
|
- logger.info('successfully logged in, saving cookies to file...');
|
|
|
- fs.writeFileSync(path.resolve(this.webshotCookiesLockfile), JSON.stringify(cookies, null, 2), 'utf-8');
|
|
|
+ ScreenNameNormalizer._queryUser = this.queryUser;
|
|
|
+ sendAllStories = (rawUserName, receiver) => {
|
|
|
+ const sender = this.sendStories(`instagram stories for ${rawUserName}`, receiver);
|
|
|
+ this.queryUser(rawUserName)
|
|
|
+ .then(userNameId => {
|
|
|
+ const [userName, userId] = userNameId.split(':');
|
|
|
+ if (userName in this.cache && Object.keys(this.cache[userName].stories).length > 0) {
|
|
|
+ return Promise.resolve(
|
|
|
+ Object.values(this.cache[userName].stories)
|
|
|
+ .map(story => ({...story, user: this.cache[userName].user}))
|
|
|
+ .sort((i1, i2) => BigNumOps.compare(i2.pk, i1.pk))
|
|
|
+ );
|
|
|
+ }
|
|
|
+ return this.client.feed.reelsMedia({userIds: [userId]}).items()
|
|
|
+ .then(storyItems => {
|
|
|
+ storyItems.forEach(item => {
|
|
|
+ if (!(item.pk in this.cache[userName].stories)) {
|
|
|
+ this.cache[userName].stories[item.pk] = item;
|
|
|
+ }
|
|
|
+ });
|
|
|
+ if (storyItems.length === 0) this.bot.sendTo(receiver, `当前用户 (@${userName}) 没有可用的推特故事。`);
|
|
|
+ return storyItems;
|
|
|
+ });
|
|
|
})
|
|
|
- .catch((err: Error) => {
|
|
|
- if (err.name === 'TimeoutError') logger.warn('navigation timed out, assuming login has failed');
|
|
|
- throw err;
|
|
|
+ .then(storyItems => this.workOnMedia(storyItems, sender))
|
|
|
+ .catch((error: IgClientError & Partial<RequestError>) => {
|
|
|
+ if (error instanceof IgNetworkError) {
|
|
|
+ logger.warn(`error on fetching stories for ${rawUserName}: ${JSON.stringify(error.cause)}`);
|
|
|
+ this.bot.sendTo(receiver, `获取 Fleets 时出现错误：原因： ${error.cause}`);
|
|
|
+ } else {
|
|
|
+ logger.error(`unhandled error on fetching media for ${rawUserName}: ${error}`);
|
|
|
+ this.bot.sendTo(receiver, `获取 Fleets 时发生未知错误： ${error}`);
|
|
|
+ }
|
|
|
});
|
|
|
};
|
|
|
- ScreenNameNormalizer._queryUser = this.queryUser;
|
|
|
- const parseMediaError = (err: IgClientError) => {
|
|
|
- if (!(err instanceof IgResponseError && err.text === 'Media not found or unavailable')) {
|
|
|
- logger.warn(`error retrieving instagram media: ${err.message}`);
|
|
|
- return `获取媒体时出现错误：${err.message}`;
|
|
|
- }
|
|
|
- return '找不到请求的媒体，它可能已被删除。';
|
|
|
- };
|
|
|
- getPostOwner = (segmentId) =>
|
|
|
- this.client.media.info(urlSegmentToId(segmentId))
|
|
|
- .then(media => media.items[0].user)
|
|
|
- .then(user => `${user.username}:${user.pk}`)
|
|
|
- .catch((err: IgClientError) => { throw Error(parseMediaError(err)); });
|
|
|
- sendPost = (segmentId, receiver) => {
|
|
|
- this.getMedia(segmentId, this.sendMedia(`instagram media ${segmentId}`, receiver))
|
|
|
- .catch((err: IgClientError) => { this.bot.sendTo(receiver, parseMediaError(err)); });
|
|
|
- };
|
|
|
}
|
|
|
|
|
|
public launch = () => {
|
|
|
this.webshot = new Webshot(
|
|
|
this.wsUrl,
|
|
|
this.mode,
|
|
|
- () => this.webshotCookies,
|
|
|
- () => setTimeout(this.work, this.workInterval * 1000)
|
|
|
+ () => {
|
|
|
+ setTimeout(this.workForAll, this.workInterval * 1000);
|
|
|
+ setTimeout(() => {
|
|
|
+ this.work();
|
|
|
+ setInterval(this.workForAll, this.workInterval * 10000);
|
|
|
+ }, this.workInterval * 1200);
|
|
|
+ }
|
|
|
);
|
|
|
};
|
|
|
|
|
|
- public queryUser = (username: string) => this.client.user.searchExact(username)
|
|
|
- .then(user => `${user.username}:${user.pk}`);
|
|
|
+ public queryUser = (rawUserName: string) => {
|
|
|
+ const username = ScreenNameNormalizer.normalize(rawUserName).split(':')[0];
|
|
|
+ if (username in this.cache) {
|
|
|
+ return Promise.resolve(`${username}:${this.cache[username].user.pk}`);
|
|
|
+ }
|
|
|
+ return this.client.user.searchExact(username)
|
|
|
+ .then(user => {
|
|
|
+ this.cache[user.username] = {user, stories: {}};
|
|
|
+ return `${user.username}:${user.pk}`;
|
|
|
+ });
|
|
|
+ };
|
|
|
|
|
|
private workOnMedia = (
|
|
|
mediaItems: MediaItem[],
|
|
|
sendMedia: (msg: string, text: string, author: string) => void
|
|
|
) => this.webshot(mediaItems, sendMedia, this.webshotDelay);
|
|
|
|
|
|
- public urlSegmentToId = urlSegmentToId;
|
|
|
-
|
|
|
- public getMedia = (segmentId: string, sender: (msg: string, text: string, author: string) => void) =>
|
|
|
- this.client.media.info(urlSegmentToId(segmentId))
|
|
|
- .then(media => {
|
|
|
- const mediaItem = media.items[0] as MediaItem;
|
|
|
- logger.debug(`api returned media post ${JSON.stringify(mediaItem)} for query id=${segmentId}`);
|
|
|
- return this.workOnMedia([mediaItem], sender);
|
|
|
- });
|
|
|
-
|
|
|
- private sendMedia = (source?: string, ...to: IChat[]) => (msg: string, text: string, author: string) => {
|
|
|
+ private sendStories = (source?: string, ...to: IChat[]) => (msg: string, text: string, author: string) => {
|
|
|
to.forEach(subscriber => {
|
|
|
logger.info(`pushing data${source ? ` of ${Message.ellipseBase64(source)}` : ''} to ${JSON.stringify(subscriber)}`);
|
|
|
retryOnError(
|
|
|
@@ -275,13 +257,52 @@ export default class {
|
|
|
});
|
|
|
};
|
|
|
|
|
|
+ private cache: {
|
|
|
+ [userName: string]: {
|
|
|
+ user: UserFeedResponseUser,
|
|
|
+ stories: {[storyId: string]: MediaItem},
|
|
|
+ },
|
|
|
+ } = {};
|
|
|
+
|
|
|
+ private workForAll = () => {
|
|
|
+ const idToUserMap: {[id: number]: UserFeedResponseUser} = {};
|
|
|
+ Promise.all(Object.entries(this.lock.threads).map(entry => {
|
|
|
+ const id = entry[1].id;
|
|
|
+ const userName = parseLink(entry[0]).userName;
|
|
|
+ logger.debug(`preparing to add user @${userName} to next pull task...`);
|
|
|
+ if (userName in this.cache) return Promise.resolve(idToUserMap[id] = this.cache[userName].user);
|
|
|
+ return this.client.user.info(id).then(user => {
|
|
|
+ logger.debug(`initialized cache item for user ${user.full_name} (@${userName})`);
|
|
|
+ this.cache[userName] = {user, stories: {}};
|
|
|
+ return idToUserMap[id] = user as UserFeedResponseUser;
|
|
|
+ });
|
|
|
+ }))
|
|
|
+ .then(() => {
|
|
|
+ logger.debug(`pulling stories for users: ${Object.values(idToUserMap).map(user => user.username)}`);
|
|
|
+ this.client.feed.reelsMedia({
|
|
|
+ userIds: Object.keys(idToUserMap),
|
|
|
+ }).items()
|
|
|
+ .then(storyItems => storyItems.forEach(item => {
|
|
|
+ if (!(item.pk in this.cache[idToUserMap[item.user.pk].username].stories)) {
|
|
|
+ this.cache[idToUserMap[item.user.pk].username].stories[item.pk] = item;
|
|
|
+ }
|
|
|
+ }))
|
|
|
+ .catch((error: IgClientError & Partial<RequestError>) => {
|
|
|
+ if (error instanceof IgNetworkError) {
|
|
|
+ logger.warn(`error on fetching stories for all: ${JSON.stringify(error.cause)}`);
|
|
|
+ } else {
|
|
|
+ logger.error(`unhandled error on fetching media for all: ${error}`);
|
|
|
+ }
|
|
|
+ });
|
|
|
+ });
|
|
|
+ };
|
|
|
+
|
|
|
public work = () => {
|
|
|
const lock = this.lock;
|
|
|
+ logger.debug(`current cache: ${JSON.stringify(this.cache)}`);
|
|
|
if (this.workInterval < 1) this.workInterval = 1;
|
|
|
if (lock.feed.length === 0) {
|
|
|
- setTimeout(() => {
|
|
|
- this.work();
|
|
|
- }, this.workInterval * 1000);
|
|
|
+ setTimeout(this.work, this.workInterval * 1000);
|
|
|
return;
|
|
|
}
|
|
|
if (lock.workon >= lock.feed.length) lock.workon = 0;
|
|
|
@@ -297,35 +318,23 @@ export default class {
|
|
|
}
|
|
|
|
|
|
const currentFeed = lock.feed[lock.workon];
|
|
|
- logger.debug(`pulling feed ${currentFeed}`);
|
|
|
+ logger.debug(`searching for new items from ${currentFeed} in cache`);
|
|
|
|
|
|
- const promise = new Promise<UserFeedResponseItemsItem[]>(resolve => {
|
|
|
+ const promise = new Promise<MediaItem[]>(resolve => {
|
|
|
const match = /https:\/\/www\.instagram\.com\/([^\/]+)/.exec(currentFeed);
|
|
|
if (match) {
|
|
|
- const feed = this.client.feed.user(lock.threads[currentFeed].id);
|
|
|
- const newer = (item: UserFeedResponseItemsItem) =>
|
|
|
+ const cachedFeed = this.cache[match[1]];
|
|
|
+ if (!cachedFeed) {
|
|
|
+ setTimeout(this.work, this.workInterval * 1000);
|
|
|
+ resolve([]);
|
|
|
+ }
|
|
|
+ const newer = (item: MediaItem) =>
|
|
|
BigNumOps.compare(item.pk, lock.threads[currentFeed].offset) > 0;
|
|
|
- const fetchMore = () => new Promise<UserFeedResponseItemsItem[]>(fetch => {
|
|
|
- feed.request().then(response => {
|
|
|
- if (response.items.length === 0) return fetch([]);
|
|
|
- if (response.items.every(newer)) {
|
|
|
- fetchMore().then(fetched => fetch(response.items.concat(fetched)));
|
|
|
- } else fetch(response.items.filter(newer));
|
|
|
- }, (error: IgClientError & Partial<RequestError>) => {
|
|
|
- if (error instanceof IgNetworkError) {
|
|
|
- logger.warn(`error on fetching media for ${currentFeed}: ${JSON.stringify(error.cause)}`);
|
|
|
- if (!(error instanceof IgNotFoundError)) return;
|
|
|
- lock.threads[currentFeed].subscribers.forEach(subscriber => {
|
|
|
- logger.info(`sending notfound message of ${currentFeed} to ${JSON.stringify(subscriber)}`);
|
|
|
- this.bot.sendTo(subscriber, `链接 ${currentFeed} 指向的用户或列表不存在，请退订。`).catch();
|
|
|
- });
|
|
|
- } else {
|
|
|
- logger.error(`unhandled error on fetching media for ${currentFeed}: ${JSON.stringify(error)}`);
|
|
|
- }
|
|
|
- fetch([]);
|
|
|
- });
|
|
|
- });
|
|
|
- fetchMore().then(resolve);
|
|
|
+ resolve(Object.values(cachedFeed.stories)
|
|
|
+ .filter(newer)
|
|
|
+ .map(story => ({...story, user: cachedFeed.user}))
|
|
|
+ .sort((i1, i2) => BigNumOps.compare(i2.pk, i1.pk))
|
|
|
+ );
|
|
|
}
|
|
|
});
|
|
|
|
|
|
@@ -341,7 +350,7 @@ export default class {
|
|
|
if (currentThread.offset === '-1') { updateOffset(); return; }
|
|
|
if (currentThread.offset === '0') mediaItems.splice(1);
|
|
|
|
|
|
- return this.workOnMedia(mediaItems, this.sendMedia(`thread ${currentFeed}`, ...currentThread.subscribers))
|
|
|
+ return this.workOnMedia(mediaItems, this.sendStories(`thread ${currentFeed}`, ...currentThread.subscribers))
|
|
|
.then(updateDate).then(updateOffset);
|
|
|
})
|
|
|
.then(() => {
|
