Skip to content

Commit

Permalink
perf: speedup bank payment query (#637)
Browse files Browse the repository at this point in the history 
* perf: parallelize bank payment requests

* feat: test with parellize loading of 3 connections

* revert: revert parellized fetch

* fix: improve ui

* feat: divider and parellize

* fix: fix divider and recent 3 months

* fix: fix partial loading

* chore: adjust details
  • Loading branch information
@SunnyCloudYang
SunnyCloudYang authored Oct 30, 2024
1 parent 4bec1e5 commit 94d0a3c
Show file tree
Hide file tree
Showing 5 changed files with 126 additions and 57 deletions.
3 changes: 3 additions & 0 deletions apps/thu-info-app/src/assets/translations/en.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -609,4 +609,7 @@ export default {
"Service provider has stopped maintaining status for new washers, so any feedback about missing washing machines will not be accepted. Data in this page comes solely from \"华清捷利\" mini program, thus any feedback about extending the page's functionality will not be accepted.",
noMoreData: "No more data",
hideWeekend: "Hide Weekend",
loadAllData: "Load all data",
recentThreeMonths: "Recent 3 months",
noData: "No data",
};
3 changes: 3 additions & 0 deletions apps/thu-info-app/src/assets/translations/zh.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -575,4 +575,7 @@ export default {
"厂家已停止对新洗衣机的维护，因此我们不接受对缺少洗衣机的反馈。此外，本系统数据完全来源于“华清捷利”小程序，因此我们不接受有关扩展功能的反馈。",
noMoreData: "没有更多数据了~",
hideWeekend: "隐藏周末",
loadAllData: "加载全部",
recentThreeMonths: "最近三个月",
noData: "暂无数据",
};
43 changes: 34 additions & 9 deletions apps/thu-info-app/src/ui/home/bankPayment.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
Text,
TouchableOpacity,
View,
StyleSheet,
} from "react-native";
import Snackbar from "react-native-snackbar";
import {getStr} from "../../utils/i18n";
Expand All @@ -26,6 +27,8 @@ export const BankPaymentScreen = () => {
const [foundation, setFoundation] = useState(false);
const [modalOpen, setModalOpen] = useState(false);

const [loadPartial, setLoadPartial] = useState(true);

const headerHeight = useHeaderHeight();

const themeName = useColorScheme();
Expand All @@ -43,7 +46,7 @@ export const BankPaymentScreen = () => {
let cancelled = false;

helper
.getBankPayment(foundation)
.getBankPayment(foundation, loadPartial)
.then((r) => {
if (cancelled) {
return;
Expand All @@ -65,7 +68,7 @@ export const BankPaymentScreen = () => {
};
};

useEffect(fetchData, [foundation]);
useEffect(fetchData, [foundation, loadPartial]);

return (
<View style={{flex: 1}}>
Expand Down Expand Up @@ -161,7 +164,10 @@ export const BankPaymentScreen = () => {
/>
}>
<View>
{data.map(({month, payment}) => (
<Text style={{ fontSize: 12, color: colors.fontB2, marginTop: 8, marginStart: 8 }}>
{loadPartial ? getStr("recentThreeMonths") : getStr("all")}
</Text>
{data.length ? data.map(({month, payment}) => (
<View key={month} style={{marginTop: 12}}>
<View
style={{
Expand Down Expand Up @@ -197,8 +203,8 @@ export const BankPaymentScreen = () => {
{index > 0 && (
<View
style={{
borderWidth: 0.4,
borderColor: colors.themeGrey,
borderBottomColor: colors.themeGrey,
borderBottomWidth: StyleSheet.hairlineWidth,
margin: 12,
}}
/>
Expand Down Expand Up @@ -263,17 +269,36 @@ export const BankPaymentScreen = () => {
))}
</RoundedView>
</View>
))}
)) : (
<RoundedView
style={{
marginTop: 12,
padding: 12,
alignItems: "center",
}}>
<Text
style={{
color: colors.fontB2,
fontSize: 14,
textAlign: "center",
marginVertical: 12,
}}>
{refreshing ? " " : getStr("noData")}
</Text>
</RoundedView>
)}
</View>
<View>
<Text
style={{
color: colors.fontB2,
color: (refreshing || !loadPartial) ? colors.fontB2 : colors.themeLightPurple,
fontSize: 12,
textAlign: "center",
marginVertical: 12,
}}>
{refreshing ? getStr("loading") : getStr("noMoreData")}
}}
onPress={() => loadPartial && setLoadPartial(false)}
>
{refreshing ? getStr("loading") : loadPartial ? getStr("loadAllData") : getStr("noMoreData")}
</Text>
</View>
</ScrollView>
Expand Down
5 changes: 3 additions & 2 deletions packages/thu-info-lib/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,15 @@ import {
getPhysicalExamResult,
getReport,
postAssessmentForm,
getBankPayment,
// getBankPayment,
getCalendar,
getInvoiceList,
getInvoicePDF,
switchLang,
naiveSendMail,
getCalendarImageUrl, getSchoolCalendarYear,
getGraduateIncome,
getBankPaymentParellize,
} from "./lib/basics";
import {forgetDevice, login, logout} from "./lib/core";
import {getDormScore, getElePayRecord, getEleRechargePayCode, getEleRemainder, resetDormPassword} from "./lib/dorm";
Expand Down Expand Up @@ -407,7 +408,7 @@ export class InfoHelper {
* Get the bank payment records of the user.
* @param foundation whether to get bank payment result by 基金会 or not
*/
public getBankPayment = async (foundation = false): Promise<BankPaymentByMonth[]> => getBankPayment(this, foundation);
public getBankPayment = async (foundation = false, loadPartial = false): Promise<BankPaymentByMonth[]> => getBankPaymentParellize(this, foundation, loadPartial);

/**
* Get the graduate income records of the user according to the date range
Expand Down
129 changes: 83 additions & 46 deletions packages/thu-info-lib/src/lib/basics.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,7 @@ export const getInvoicePDF = (helper: InfoHelper, busNumber: string): Promise<st
export const getBankPayment = async (
helper: InfoHelper,
foundation: boolean,
loadPartial: boolean = false,
): Promise<BankPaymentByMonth[]> =>
roamingWrapperWithMocks(
helper,
Expand All @@ -516,56 +517,92 @@ export const getBankPayment = async (
if (options.length === 0) {
return [];
}
const form = options.map((o) => `year=${encodeURIComponent(o)}`).join("&");
const form = (loadPartial ? options.slice(0, Math.min(3, options.length)) : options).map((o) => `year=${encodeURIComponent(o)}`).join("&");
const result = await uFetch(foundation ? FOUNDATION_BANK_PAYMENT_SEARCH_URL : BANK_PAYMENT_SEARCH_URL, form as never as object, 60000, "UTF-8", true);
const $ = cheerio.load(result);
const titles = $("div strong")
.map((_, e) => {
const titleElement = e as TagElement;
const text = (titleElement.children[0] as DataNode).data?.trim();
if (text === undefined) {
return undefined;
}
const res = /(\d+年\d+月)银行代发结果/g.exec(text);
if (res === null || res[1] === undefined) {
return undefined;
}
if (((titleElement.parentNode?.next?.next as TagElement)?.firstChild as TagElement)?.name !== "table") {
return undefined;
}
return res[1];
})
.get()
.filter((text) => text !== undefined) as string[];
return $("div table tbody")
.filter(index => index < titles.length)
.map((index, e) => {
const rows = cheerio.load(e)("tr");
const data = rows.slice(1, rows.length - 1);
return {
month: titles[index],
payment: data.map((_, row) => {
const columns = cheerio.load(row)("td");
return {
department: getCheerioText(columns[1], 0),
project: getCheerioText(columns[2], 0),
usage: getCheerioText(columns[3], 0),
description: getCheerioText(columns[4], 0),
bank: getCheerioText(columns[5], 0),
time: getCheerioText(columns[6], 0),
total: getCheerioText((columns[7] as TagElement).children[0], 0),
deduction: getCheerioText((columns[8] as TagElement).children[0], 0),
actual: getCheerioText((columns[9] as TagElement).children[0], 0),
deposit: getCheerioText((columns[10] as TagElement).children[0], 0),
cash: getCheerioText((columns[11] as TagElement).children[0], 0),
} as BankPayment;
}).get().reverse(),
};
})
.get() as BankPaymentByMonth[];

return parseAndFilterBankPayment(result);
},
MOCK_BANK_PAYMENT,
);

export const getBankPaymentParellize = async (
helper: InfoHelper,
foundation: boolean,
loadPartial: boolean = false,
): Promise<BankPaymentByMonth[]> => {
const PARTIAL_NUM = 3;
const MAX_PARALLEL_TASKS = 3;
return roamingWrapperWithMocks(
helper,
"default",
foundation ? "C1ADD6B60D050B64E0C7B8F195CE89EC" : "2A5182CB3F36E80395FC2091001BDEA6",
async (s) => {
if (s === undefined) {
throw new LibError();
}
const options = cheerio.load(s)("option").map((_, e) => (e as TagElement).attribs.value).get();
if (options.length === 0) {
return [];
}

const loadOptions = (loadPartial ? options.slice(0, PARTIAL_NUM) : options).map(o => `year=${encodeURIComponent(o)}`);
const jointOptions = [];
for (let i = 0; i < MAX_PARALLEL_TASKS; i++) {
jointOptions.push(loadOptions.slice(i * Math.ceil(loadOptions.length / MAX_PARALLEL_TASKS), (i + 1) * Math.ceil(loadOptions.length / MAX_PARALLEL_TASKS)).join("&"));
}

const requests = jointOptions.filter(it => it !== "").map((o) => {
return uFetch(foundation ? FOUNDATION_BANK_PAYMENT_SEARCH_URL : BANK_PAYMENT_SEARCH_URL, o as never as object, 60000, "UTF-8", true);
});
const results = await Promise.all(requests);
const parsedResults = results.map((result) => {
return parseAndFilterBankPayment(result);
}).flatMap((it) => it);
return parsedResults;
},
MOCK_BANK_PAYMENT,
);
};

const parseAndFilterBankPayment = (html: string) => {
const $ = cheerio.load(html);
const titleElements = $("div strong");
return titleElements.map((_, e) => {
const titleElement = e as TagElement;
const text = (titleElement.children[0] as DataNode).data?.trim();
if (text === undefined) {
return undefined;
}
const res = /(\d+年\d+月)银行代发结果/g.exec(text);
if (res === null || res[1] === undefined) {
return undefined;
}
if (((titleElement.parentNode?.next?.next as TagElement)?.firstChild as TagElement)?.name !== "table") {
return undefined;
}
const data = cheerio.load((titleElement.parent?.next?.next as TagElement)?.firstChild as cheerio.AnyNode)("tbody tr").slice(1, -1);
const payment = data.map((__, row) => {
const columns = cheerio.load(row)("td");
return {
department: getCheerioText(columns[1], 0),
project: getCheerioText(columns[2], 0),
usage: getCheerioText(columns[3], 0),
description: getCheerioText(columns[4], 0),
bank: getCheerioText(columns[5], 0),
time: getCheerioText(columns[6], 0),
total: getCheerioText((columns[7] as TagElement).children[0], 0),
deduction: getCheerioText((columns[8] as TagElement).children[0], 0),
actual: getCheerioText((columns[9] as TagElement).children[0], 0),
deposit: getCheerioText((columns[10] as TagElement).children[0], 0),
cash: getCheerioText((columns[11] as TagElement).children[0], 0),
} as BankPayment;
}).get().reverse();
return {
month: res[1],
payment,
};
}).get().filter((it) => it !== undefined) as BankPaymentByMonth[];
};

export const getGraduateIncome = async (
helper: InfoHelper,
Expand Down

0 comments on commit 94d0a3c

Please sign in to comment.