Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

perf: speedup bank payment query #637

Merged
merged 8 commits into from
Oct 30, 2024
Merged

perf: speedup bank payment query #637

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
032940d
perf: parallelize bank payment requests
SunnyCloudYang Oct 27, 2024
3ea8a3c
feat: test with parellize loading of 3 connections
SunnyCloudYang Oct 30, 2024
c5f41b6
revert: revert parellized fetch
SunnyCloudYang Oct 30, 2024
a14b7b3
fix: improve ui
SunnyCloudYang Oct 30, 2024
37bc98c
feat: divider and parellize
SunnyCloudYang Oct 30, 2024
3deea79
fix: fix divider and recent 3 months
SunnyCloudYang Oct 30, 2024
da93416
fix: fix partial loading
SunnyCloudYang Oct 30, 2024
6071a84
chore: adjust details
SunnyCloudYang Oct 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions apps/thu-info-app/src/assets/translations/en.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -609,4 +609,7 @@ export default {
"Service provider has stopped maintaining status for new washers, so any feedback about missing washing machines will not be accepted. Data in this page comes solely from \"华清捷利\" mini program, thus any feedback about extending the page's functionality will not be accepted.",
noMoreData: "No more data",
hideWeekend: "Hide Weekend",
loadAllData: "Load all data",
recentThreeMonths: "Recent 3 months",
noData: "No data",
};
3 changes: 3 additions & 0 deletions apps/thu-info-app/src/assets/translations/zh.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -575,4 +575,7 @@ export default {
"厂家已停止对新洗衣机的维护，因此我们不接受对缺少洗衣机的反馈。此外，本系统数据完全来源于“华清捷利”小程序，因此我们不接受有关扩展功能的反馈。",
noMoreData: "没有更多数据了~",
hideWeekend: "隐藏周末",
loadAllData: "加载全部",
recentThreeMonths: "最近三个月",
noData: "暂无数据",
};
43 changes: 34 additions & 9 deletions apps/thu-info-app/src/ui/home/bankPayment.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
Text,
TouchableOpacity,
View,
StyleSheet,
} from "react-native";
import Snackbar from "react-native-snackbar";
import {getStr} from "../../utils/i18n";
Expand All @@ -26,6 +27,8 @@ export const BankPaymentScreen = () => {
const [foundation, setFoundation] = useState(false);
const [modalOpen, setModalOpen] = useState(false);

const [loadPartial, setLoadPartial] = useState(true);

const headerHeight = useHeaderHeight();

const themeName = useColorScheme();
Expand All @@ -43,7 +46,7 @@ export const BankPaymentScreen = () => {
let cancelled = false;

helper
.getBankPayment(foundation)
.getBankPayment(foundation, loadPartial)
.then((r) => {
if (cancelled) {
return;
Expand All @@ -65,7 +68,7 @@ export const BankPaymentScreen = () => {
};
};

useEffect(fetchData, [foundation]);
useEffect(fetchData, [foundation, loadPartial]);

return (
<View style={{flex: 1}}>
Expand Down Expand Up @@ -161,7 +164,10 @@ export const BankPaymentScreen = () => {
/>
}>
<View>
{data.map(({month, payment}) => (
<Text style={{ fontSize: 12, color: colors.fontB2, marginTop: 8, marginStart: 8 }}>
{loadPartial ? getStr("recentThreeMonths") : getStr("all")}
</Text>
{data.length ? data.map(({month, payment}) => (
<View key={month} style={{marginTop: 12}}>
<View
style={{
Expand Down Expand Up @@ -197,8 +203,8 @@ export const BankPaymentScreen = () => {
{index > 0 && (
<View
style={{
borderWidth: 0.4,
borderColor: colors.themeGrey,
borderBottomColor: colors.themeGrey,
borderBottomWidth: StyleSheet.hairlineWidth,
margin: 12,
}}
/>
Expand Down Expand Up @@ -263,17 +269,36 @@ export const BankPaymentScreen = () => {
))}
</RoundedView>
</View>
))}
)) : (
<RoundedView
style={{
marginTop: 12,
padding: 12,
alignItems: "center",
}}>
<Text
style={{
color: colors.fontB2,
fontSize: 14,
textAlign: "center",
marginVertical: 12,
}}>
{refreshing ? " " : getStr("noData")}
</Text>
</RoundedView>
)}
</View>
<View>
<Text
style={{
color: colors.fontB2,
color: (refreshing || !loadPartial) ? colors.fontB2 : colors.themeLightPurple,
fontSize: 12,
textAlign: "center",
marginVertical: 12,
}}>
{refreshing ? getStr("loading") : getStr("noMoreData")}
}}
onPress={() => loadPartial && setLoadPartial(false)}
>
{refreshing ? getStr("loading") : loadPartial ? getStr("loadAllData") : getStr("noMoreData")}
</Text>
</View>
</ScrollView>
Expand Down
5 changes: 3 additions & 2 deletions packages/thu-info-lib/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,15 @@ import {
getPhysicalExamResult,
getReport,
postAssessmentForm,
getBankPayment,
// getBankPayment,
getCalendar,
getInvoiceList,
getInvoicePDF,
switchLang,
naiveSendMail,
getCalendarImageUrl, getSchoolCalendarYear,
getGraduateIncome,
getBankPaymentParellize,
} from "./lib/basics";
import {forgetDevice, login, logout} from "./lib/core";
import {getDormScore, getElePayRecord, getEleRechargePayCode, getEleRemainder, resetDormPassword} from "./lib/dorm";
Expand Down Expand Up @@ -407,7 +408,7 @@ export class InfoHelper {
* Get the bank payment records of the user.
* @param foundation whether to get bank payment result by 基金会 or not
*/
public getBankPayment = async (foundation = false): Promise<BankPaymentByMonth[]> => getBankPayment(this, foundation);
public getBankPayment = async (foundation = false, loadPartial = false): Promise<BankPaymentByMonth[]> => getBankPaymentParellize(this, foundation, loadPartial);

/**
* Get the graduate income records of the user according to the date range
Expand Down
129 changes: 83 additions & 46 deletions packages/thu-info-lib/src/lib/basics.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,7 @@ export const getInvoicePDF = (helper: InfoHelper, busNumber: string): Promise<st
export const getBankPayment = async (
helper: InfoHelper,
foundation: boolean,
loadPartial: boolean = false,
): Promise<BankPaymentByMonth[]> =>
roamingWrapperWithMocks(
helper,
Expand All @@ -516,56 +517,92 @@ export const getBankPayment = async (
if (options.length === 0) {
return [];
}
const form = options.map((o) => `year=${encodeURIComponent(o)}`).join("&");
const form = (loadPartial ? options.slice(0, Math.min(3, options.length)) : options).map((o) => `year=${encodeURIComponent(o)}`).join("&");
const result = await uFetch(foundation ? FOUNDATION_BANK_PAYMENT_SEARCH_URL : BANK_PAYMENT_SEARCH_URL, form as never as object, 60000, "UTF-8", true);
const $ = cheerio.load(result);
const titles = $("div strong")
.map((_, e) => {
const titleElement = e as TagElement;
const text = (titleElement.children[0] as DataNode).data?.trim();
if (text === undefined) {
return undefined;
}
const res = /(\d+年\d+月)银行代发结果/g.exec(text);
if (res === null || res[1] === undefined) {
return undefined;
}
if (((titleElement.parentNode?.next?.next as TagElement)?.firstChild as TagElement)?.name !== "table") {
return undefined;
}
return res[1];
})
.get()
.filter((text) => text !== undefined) as string[];
return $("div table tbody")
.filter(index => index < titles.length)
.map((index, e) => {
const rows = cheerio.load(e)("tr");
const data = rows.slice(1, rows.length - 1);
return {
month: titles[index],
payment: data.map((_, row) => {
const columns = cheerio.load(row)("td");
return {
department: getCheerioText(columns[1], 0),
project: getCheerioText(columns[2], 0),
usage: getCheerioText(columns[3], 0),
description: getCheerioText(columns[4], 0),
bank: getCheerioText(columns[5], 0),
time: getCheerioText(columns[6], 0),
total: getCheerioText((columns[7] as TagElement).children[0], 0),
deduction: getCheerioText((columns[8] as TagElement).children[0], 0),
actual: getCheerioText((columns[9] as TagElement).children[0], 0),
deposit: getCheerioText((columns[10] as TagElement).children[0], 0),
cash: getCheerioText((columns[11] as TagElement).children[0], 0),
} as BankPayment;
}).get().reverse(),
};
})
.get() as BankPaymentByMonth[];

return parseAndFilterBankPayment(result);
},
MOCK_BANK_PAYMENT,
);

export const getBankPaymentParellize = async (
helper: InfoHelper,
foundation: boolean,
loadPartial: boolean = false,
): Promise<BankPaymentByMonth[]> => {
const PARTIAL_NUM = 3;
const MAX_PARALLEL_TASKS = 3;
return roamingWrapperWithMocks(
helper,
"default",
foundation ? "C1ADD6B60D050B64E0C7B8F195CE89EC" : "2A5182CB3F36E80395FC2091001BDEA6",
async (s) => {
if (s === undefined) {
throw new LibError();
}
const options = cheerio.load(s)("option").map((_, e) => (e as TagElement).attribs.value).get();
if (options.length === 0) {
return [];
}

const loadOptions = (loadPartial ? options.slice(0, PARTIAL_NUM) : options).map(o => `year=${encodeURIComponent(o)}`);
const jointOptions = [];
for (let i = 0; i < MAX_PARALLEL_TASKS; i++) {
jointOptions.push(loadOptions.slice(i * Math.ceil(loadOptions.length / MAX_PARALLEL_TASKS), (i + 1) * Math.ceil(loadOptions.length / MAX_PARALLEL_TASKS)).join("&"));
}

const requests = jointOptions.filter(it => it !== "").map((o) => {
return uFetch(foundation ? FOUNDATION_BANK_PAYMENT_SEARCH_URL : BANK_PAYMENT_SEARCH_URL, o as never as object, 60000, "UTF-8", true);
});
const results = await Promise.all(requests);
const parsedResults = results.map((result) => {
return parseAndFilterBankPayment(result);
}).flatMap((it) => it);
return parsedResults;
},
MOCK_BANK_PAYMENT,
);
};

const parseAndFilterBankPayment = (html: string) => {
const $ = cheerio.load(html);
const titleElements = $("div strong");
return titleElements.map((_, e) => {
const titleElement = e as TagElement;
const text = (titleElement.children[0] as DataNode).data?.trim();
if (text === undefined) {
return undefined;
}
const res = /(\d+年\d+月)银行代发结果/g.exec(text);
if (res === null || res[1] === undefined) {
return undefined;
}
if (((titleElement.parentNode?.next?.next as TagElement)?.firstChild as TagElement)?.name !== "table") {
return undefined;
}
const data = cheerio.load((titleElement.parent?.next?.next as TagElement)?.firstChild as cheerio.AnyNode)("tbody tr").slice(1, -1);
const payment = data.map((__, row) => {
const columns = cheerio.load(row)("td");
return {
department: getCheerioText(columns[1], 0),
project: getCheerioText(columns[2], 0),
usage: getCheerioText(columns[3], 0),
description: getCheerioText(columns[4], 0),
bank: getCheerioText(columns[5], 0),
time: getCheerioText(columns[6], 0),
total: getCheerioText((columns[7] as TagElement).children[0], 0),
deduction: getCheerioText((columns[8] as TagElement).children[0], 0),
actual: getCheerioText((columns[9] as TagElement).children[0], 0),
deposit: getCheerioText((columns[10] as TagElement).children[0], 0),
cash: getCheerioText((columns[11] as TagElement).children[0], 0),
} as BankPayment;
}).get().reverse();
return {
month: res[1],
payment,
};
}).get().filter((it) => it !== undefined) as BankPaymentByMonth[];
};

export const getGraduateIncome = async (
helper: InfoHelper,
Expand Down
Loading