From 4a623226ad6a632bb1799dadd9b23c2470044f98 Mon Sep 17 00:00:00 2001 From: DM Date: Tue, 31 Oct 2023 03:20:15 +0800 Subject: [PATCH] feat: fetch all platform following social activities --- packages/site/src/pages/monitor/list.tsx | 2 +- packages/site/src/utils/snap.ts | 11 ++- packages/snap/snap.manifest.json | 2 +- packages/snap/src/crossbel.ts | 84 ++++++++++++++++++++- packages/snap/src/farcaster.ts | 77 ++++++++++++++++++- packages/snap/src/fetch.ts | 95 +++++++++++++++++++++++- packages/snap/src/index.ts | 8 +- packages/snap/src/lens.ts | 67 ++++++++++++++++- packages/snap/src/state.ts | 1 + 9 files changed, 330 insertions(+), 17 deletions(-) diff --git a/packages/site/src/pages/monitor/list.tsx b/packages/site/src/pages/monitor/list.tsx index 42464bc..a577949 100644 --- a/packages/site/src/pages/monitor/list.tsx +++ b/packages/site/src/pages/monitor/list.tsx @@ -30,7 +30,7 @@ const MonitorList = () => { const [monitorList, setMonitorList] = useState([]); const handleGetState = async () => { const snapState = await sendGetState(); - console.log(snapState.monitor); + console.log(snapState); // toast({ // title: 'You submitted the following values:', // description: ( diff --git a/packages/site/src/utils/snap.ts b/packages/site/src/utils/snap.ts index 9d8ec1b..8f2805a 100644 --- a/packages/site/src/utils/snap.ts +++ b/packages/site/src/utils/snap.ts @@ -58,12 +58,19 @@ export type SocialActivity = { activities: string[]; total: number; }; + +export type CronActivity = { + id: string; + text: string; + owner?: string; +}; + export type TProfile = { handle: string; address?: string; avatar?: string; - activities?: SocialActivity[]; - lastActivities?: SocialActivity[]; + activities?: CronActivity[]; + lastActivities?: CronActivity[]; }; export enum Platform { diff --git a/packages/snap/snap.manifest.json b/packages/snap/snap.manifest.json index 9f20341..0240479 100644 --- a/packages/snap/snap.manifest.json +++ b/packages/snap/snap.manifest.json @@ -7,7 +7,7 @@ "url": "https://github.com/NaturalSelectionLabs/RSS3-MetaMask-Snap.git" }, "source": { - "shasum": "4j3pb8CbYuEUdHa7PUICtBRVvCEjyPan3AwxoUujXj8=", + "shasum": "sgMIE/TzXLQiskQP4BnEoZdlDYt99Y9Wi1JAEvzHnAY=", "location": { "npm": { "filePath": "dist/bundle.js", diff --git a/packages/snap/src/crossbel.ts b/packages/snap/src/crossbel.ts index 8c48332..385c912 100644 --- a/packages/snap/src/crossbel.ts +++ b/packages/snap/src/crossbel.ts @@ -1,4 +1,7 @@ import { isValidWalletAddress } from './utils'; +import { getMultiple } from './fetch'; +// import { CronActivity } from './state'; +import { CronActivity, getState } from './state'; import { Platform, TProfile, TRelationChainResult } from '.'; const API = `https://indexer.crossbell.io/v1`; @@ -135,8 +138,14 @@ async function getCharacterId(handle: string) { * Retrieves the followers for the given character ID from the Crossbell API. * * @param id - The character ID to retrieve the followers for. + * @param handle - The Handle. + * @param timestamp - The timestamp. */ -export async function getFollowingByCharacterId(id: string) { +export async function getFollowingByCharacterId( + id: string, + handle: string, + timestamp?: string, +) { const following: TProfile[] = []; let hasNextPage = true; let cursor: string | undefined; @@ -168,7 +177,69 @@ export async function getFollowingByCharacterId(id: string) { } } - return following; + const addresses = following + .map((item) => item.address) + .filter((addr) => addr !== undefined) + .slice(0, 50) as string[]; + + // Each 50 addresses is a set of requests + const addressesGroup: string[][] = []; + for (let i = 0; i < addresses.length; i += 100) { + addressesGroup.push(addresses.slice(i, i + 100)); + } + + const groupAddresses: { + owner: string; + activities: CronActivity[]; + oldActivities: CronActivity[]; + }[] = []; + + const addressGroupPromise = addressesGroup.map(async (group) => { + const activities = await getMultiple(group, timestamp); + const executeActivitiesPromise = activities.map(async (activity) => { + const state = await getState(); + // async; + const { monitor } = state; + const cachedFollowing = monitor.find((item) => item.search === handle); + let oldActivities: CronActivity[] = []; + if (cachedFollowing?.activities) { + oldActivities = + cachedFollowing.activities.find((item) => item.address === handle) + ?.activities ?? []; + } + // activity.oldActivities = oldActivities; + return { + ...activity, + oldActivities, + }; + }); + const executeActivities = await Promise.all(executeActivitiesPromise); + groupAddresses.push(...executeActivities); + }); + + await Promise.all(addressGroupPromise); + + const fetchedFollowing = following.map((item) => { + if (item.address !== undefined) { + const findOut = groupAddresses.find((addr) => { + if (addr.owner === undefined || item.address === undefined) { + return false; + } + return addr.owner.toLowerCase() === item.address.toLowerCase(); + }); + + if (findOut) { + return { + ...item, + activities: findOut.activities, + lastActivities: findOut.oldActivities, + }; + } + } + return item; + }); + + return fetchedFollowing; } /** @@ -239,7 +310,14 @@ export async function handler( } // 2. Get following - const following = await fetchMethod(data.characterId); + + const { monitor } = await getState(); + + const timestamp = + monitor.find((item) => item.search === handle)?.latestUpdateTime ?? + undefined; + + const following = await fetchMethod(data.characterId, handle, timestamp); // 3. Return result return { diff --git a/packages/snap/src/farcaster.ts b/packages/snap/src/farcaster.ts index 607a615..ffeefd6 100644 --- a/packages/snap/src/farcaster.ts +++ b/packages/snap/src/farcaster.ts @@ -1,3 +1,5 @@ +import { getMultiple } from './fetch'; +import { CronActivity, getState } from './state'; import { Platform, TProfile, TRelationChainResult } from '.'; const API = 'https://api.warpcast.com/v2'; @@ -147,9 +149,15 @@ export async function format(data: TFarcasterUser[]): Promise { * Returns the following profiles for a given Farcaster ID. * * @param fid - The Farcaster ID to get the following profiles for. + * @param handle - The handle. + * @param timestamp - The timestamp. * @returns An array of TProfile objects representing the following profiles. */ -export async function getFollowingByFid(fid: number) { +export async function getFollowingByFid( + fid: number, + handle: string, + timestamp?: string, +) { let cursor: string | undefined; let hasNextPage = true; const following: TProfile[] = []; @@ -168,7 +176,64 @@ export async function getFollowingByFid(fid: number) { } } - return following; + const addresses = following + .map((item) => item.address) + .filter((addr) => addr !== undefined) as string[]; + + // Each 100 addresses is a set of requests + const addressesGroup: string[][] = []; + for (let i = 0; i < addresses.length; i += 100) { + addressesGroup.push(addresses.slice(i, i + 100)); + } + + const groupAddresses: { + owner: string; + activities: CronActivity[]; + oldActivities: CronActivity[]; + }[] = []; + + const addressGroupPromise = addressesGroup.map(async (group) => { + const activities = await getMultiple(group, timestamp); + const executeActivitiesPromise = activities.map(async (activity) => { + const state = await getState(); + const { monitor } = state; + const cachedFollowing = monitor.find((item) => item.search === handle); + let oldActivities: CronActivity[] = []; + if (cachedFollowing?.activities) { + oldActivities = + cachedFollowing.activities.find((item) => item.address === handle) + ?.activities ?? []; + } + return { + ...activity, + oldActivities, + }; + }); + const executeActivities = await Promise.all(executeActivitiesPromise); + groupAddresses.push(...executeActivities); + }); + + await Promise.all(addressGroupPromise); + const fetchedFollowing = following.map((item) => { + if (item.address !== undefined) { + const findOut = groupAddresses.find((addr) => { + if (addr.owner === undefined || item.address === undefined) { + return false; + } + return addr.owner.toLowerCase() === item.address.toLowerCase(); + }); + + if (findOut) { + return { + ...item, + activities: findOut.activities, + lastActivities: findOut.oldActivities, + }; + } + } + return item; + }); + return fetchedFollowing; } /** @@ -191,7 +256,13 @@ export async function handler(handle: string): Promise { }; } - const following = await getFollowingByFid(owner.fid); + const { monitor } = await getState(); + + const timestamp = + monitor.find((item) => item.search === handle)?.latestUpdateTime ?? + undefined; + + const following = await getFollowingByFid(owner.fid, handle, timestamp); return { owner: { handle: owner.handle, diff --git a/packages/snap/src/fetch.ts b/packages/snap/src/fetch.ts index 0893237..624123f 100644 --- a/packages/snap/src/fetch.ts +++ b/packages/snap/src/fetch.ts @@ -1,6 +1,6 @@ import moment from 'moment'; import { - Activity, + type Activity, formatAddressAndNS, format as sdkFormat, type Theme, @@ -76,3 +76,96 @@ export function format(activity: Activity) { }; return sdkFormat(activity, theme); } + +/** + * Get social count by rss3. + * + * @param addresses - The wallet address array. + * @param sinceTimestamp - The timestamp. + * @returns The social count array. + */ +export async function getMultiple( + addresses: (string | undefined)[], + sinceTimestamp?: string, +) { + const activities: CronActivity[] = []; + let hasNextPage = true; + let cursor: string | undefined; + + const filtedAddresses = addresses.filter( + (addr) => addr !== undefined, + ) as string[]; + + if (filtedAddresses.length === 0) { + return []; + } + + const executeAddresses = filtedAddresses; + + // 1 day ago + const timestamp = + sinceTimestamp === undefined + ? moment().subtract(1, 'day').unix() + : moment(sinceTimestamp).subtract(1, 'day').unix(); + + while (hasNextPage) { + const params = { + action_limit: 10, + limit: 500, + account: executeAddresses, + tag: ['social'], + type: ['post', 'comment'], + direction: 'out', + since_timestamp: timestamp, + cursor, + }; + const resp = await fetch( + 'https://testnet.rss3.io/data/accounts/activities', + { + method: 'POST', + headers: { + accept: 'application/json', + 'content-type': 'application/json', + }, + body: JSON.stringify(params), + }, + ); + const { data, meta } = (await resp.json()) as { + data: Activity[]; + meta: null | { cursor: string }; + }; + + data.map((item) => + activities.push({ + id: item.id, + text: format(item).join(''), + owner: item.owner, + }), + ); + + if (meta === null) { + hasNextPage = false; + } else { + cursor = meta.cursor; + } + } + + return executeAddresses.map((addr) => { + const groupBy = activities.filter( + (activity) => + activity.owner?.toLocaleLowerCase() === addr?.toLocaleLowerCase(), + ); + if (groupBy) { + return { + owner: addr, + activities: groupBy, + oldActivities: [], + }; + } + return { + owner: addr, + activities: [], + oldActivities: [], + }; + }); +} diff --git a/packages/snap/src/index.ts b/packages/snap/src/index.ts index 2687f52..cee9b7e 100644 --- a/packages/snap/src/index.ts +++ b/packages/snap/src/index.ts @@ -9,6 +9,7 @@ import { assert } from '@metamask/utils'; import { Profile } from '@rss3/js-sdk'; import moment from 'moment'; import { + CronActivity, SocialActivity, State, addAddressToState, @@ -42,8 +43,8 @@ export type TProfile = { handle: string; address?: string; avatar?: string; - activities?: SocialActivity[]; - lastActivities?: SocialActivity[]; + activities?: CronActivity[]; + lastActivities?: CronActivity[]; }; export type FetchSocialCountParams = { @@ -430,9 +431,6 @@ export const onCronjob: OnCronjobHandler = async ({ request }) => { const handles = item.profiles .filter((profile) => profile.handle !== undefined) .map((profile) => { - // const execute = executeArray.find( - // (list) => list.platform === profile.platform, - // ); if (profile.platform === Platform.Crossbell) { return { handle: profile.handle, diff --git a/packages/snap/src/lens.ts b/packages/snap/src/lens.ts index 71524ab..5a299d6 100644 --- a/packages/snap/src/lens.ts +++ b/packages/snap/src/lens.ts @@ -1,5 +1,8 @@ import { Client, cacheExchange, fetchExchange, gql } from '@urql/core'; import { isValidWalletAddress } from './utils'; +import { getMultiple } from './fetch'; +// import { CronActivity } from './state'; +import { CronActivity, getState } from './state'; import { TRelationChainResult, type TProfile, Platform } from '.'; // only need handle, ownedBy and picture. @@ -226,6 +229,68 @@ export async function handler( } } + const addresses = following + .map((item) => item.address) + .filter((addr) => addr !== undefined) as string[]; + + // Each 100 addresses is a set of requests + const addressesGroup: string[][] = []; + for (let i = 0; i < addresses.length; i += 100) { + addressesGroup.push(addresses.slice(i, i + 100)); + } + + const groupAddresses: { + owner: string; + activities: CronActivity[]; + oldActivities: CronActivity[]; + }[] = []; + const { monitor } = await getState(); + + const timestamp = + monitor.find((item) => item.search === handle)?.latestUpdateTime ?? + undefined; + const addressGroupPromise = addressesGroup.map(async (group) => { + const activities = await getMultiple(group, timestamp); + const executeActivitiesPromise = activities.map(async (activity) => { + const cachedFollowing = monitor.find((item) => item.search === handle); + let oldActivities: CronActivity[] = []; + if (cachedFollowing?.activities) { + oldActivities = + cachedFollowing.activities.find((item) => item.address === handle) + ?.activities ?? []; + } + return { + ...activity, + oldActivities, + }; + }); + + const executeActivities = await Promise.all(executeActivitiesPromise); + groupAddresses.push(...executeActivities); + }); + + await Promise.all(addressGroupPromise); + + const fetchedFollowing = following.map((item) => { + if (item.address !== undefined) { + const findOut = groupAddresses.find((addr) => { + if (addr.owner === undefined || item.address === undefined) { + return false; + } + return addr.owner.toLowerCase() === item.address.toLowerCase(); + }); + + if (findOut) { + return { + ...item, + activities: findOut.activities, + lastActivities: findOut.oldActivities, + }; + } + } + return item; + }); + return { owner: { handle: lensProfile.handle, @@ -235,6 +300,6 @@ export async function handler( platform: Platform.Lens, status: errorMessage === '', message: errorMessage || 'success', - following, + following: fetchedFollowing, }; } diff --git a/packages/snap/src/state.ts b/packages/snap/src/state.ts index 24a33a4..5219799 100644 --- a/packages/snap/src/state.ts +++ b/packages/snap/src/state.ts @@ -5,6 +5,7 @@ import { TRelationChainResult } from '.'; export type CronActivity = { id: string; text: string; + owner?: string; }; export type SocialActivity = {