Skip to content

Commit

Permalink
feat: test with parellize loading of 3 connections
Browse files Browse the repository at this point in the history
  • Loading branch information
@SunnyCloudYang
SunnyCloudYang committed Oct 30, 2024
1 parent 032940d commit 3ea8a3c
Show file tree
Hide file tree
Showing 5 changed files with 140 additions and 111 deletions.
1 change: 1 addition & 0 deletions apps/thu-info-app/src/assets/translations/en.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -609,4 +609,5 @@ export default {
"Service provider has stopped maintaining status for new washers, so any feedback about missing washing machines will not be accepted. Data in this page comes solely from \"华清捷利\" mini program, thus any feedback about extending the page's functionality will not be accepted.",
noMoreData: "No more data",
hideWeekend: "Hide Weekend",
loadAllData: "Load all data",
};
1 change: 1 addition & 0 deletions apps/thu-info-app/src/assets/translations/zh.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -575,4 +575,5 @@ export default {
"厂家已停止对新洗衣机的维护，因此我们不接受对缺少洗衣机的反馈。此外，本系统数据完全来源于“华清捷利”小程序，因此我们不接受有关扩展功能的反馈。",
noMoreData: "没有更多数据了~",
hideWeekend: "隐藏周末",
loadAllData: "加载全部数据",
};
14 changes: 9 additions & 5 deletions apps/thu-info-app/src/ui/home/bankPayment.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ export const BankPaymentScreen = () => {
const [foundation, setFoundation] = useState(false);
const [modalOpen, setModalOpen] = useState(false);

const [loadPartial, setLoadPartial] = useState(true);

const headerHeight = useHeaderHeight();

const themeName = useColorScheme();
Expand All @@ -43,7 +45,7 @@ export const BankPaymentScreen = () => {
let cancelled = false;

helper
.getBankPayment(foundation)
.getBankPayment(foundation, loadPartial)
.then((r) => {
if (cancelled) {
return;
Expand All @@ -65,7 +67,7 @@ export const BankPaymentScreen = () => {
};
};

useEffect(fetchData, [foundation]);
useEffect(fetchData, [foundation, loadPartial]);

return (
<View style={{flex: 1}}>
Expand Down Expand Up @@ -268,12 +270,14 @@ export const BankPaymentScreen = () => {
<View>
<Text
style={{
color: colors.fontB2,
color: loadPartial ? colors.themeLightPurple : colors.fontB2,
fontSize: 12,
textAlign: "center",
marginVertical: 12,
}}>
{refreshing ? getStr("loading") : getStr("noMoreData")}
}}
onPress={() => loadPartial && setLoadPartial(false)}
>
{refreshing ? getStr("loading") : loadPartial ? getStr("loadAllData") : getStr("noMoreData")}
</Text>
</View>
</ScrollView>
Expand Down
5 changes: 3 additions & 2 deletions packages/thu-info-lib/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,15 @@ import {
getPhysicalExamResult,
getReport,
postAssessmentForm,
getBankPayment,
// getBankPaymentParellize,
getCalendar,
getInvoiceList,
getInvoicePDF,
switchLang,
naiveSendMail,
getCalendarImageUrl, getSchoolCalendarYear,
getGraduateIncome,
getBankPaymentParellize,
} from "./lib/basics";
import {forgetDevice, login, logout} from "./lib/core";
import {getDormScore, getElePayRecord, getEleRechargePayCode, getEleRemainder, resetDormPassword} from "./lib/dorm";
Expand Down Expand Up @@ -407,7 +408,7 @@ export class InfoHelper {
* Get the bank payment records of the user.
* @param foundation whether to get bank payment result by 基金会 or not
*/
public getBankPayment = async (foundation = false): Promise<BankPaymentByMonth[]> => getBankPayment(this, foundation);
public getBankPayment = async (foundation = false, loadPartial = false): Promise<BankPaymentByMonth[]> => getBankPaymentParellize(this, foundation, loadPartial);

/**
* Get the graduate income records of the user according to the date range
Expand Down
230 changes: 126 additions & 104 deletions packages/thu-info-lib/src/lib/basics.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -500,76 +500,77 @@ export const getInvoicePDF = (helper: InfoHelper, busNumber: string): Promise<st
SAMPLE_INVOICE_BASE64,
);

// export const getBankPayment = async (
// helper: InfoHelper,
// foundation: boolean,
// ): Promise<BankPaymentByMonth[]> =>
// roamingWrapperWithMocks(
// helper,
// "default",
// foundation ? "C1ADD6B60D050B64E0C7B8F195CE89EC" : "2A5182CB3F36E80395FC2091001BDEA6",
// async (s) => {
// if (s === undefined) {
// throw new LibError();
// }
// const options = cheerio.load(s)("option").map((_, e) => (e as TagElement).attribs.value).get();
// if (options.length === 0) {
// return [];
// }
// const form = options.map((o) => `year=${encodeURIComponent(o)}`).join("&");
// const result = await uFetch(foundation ? FOUNDATION_BANK_PAYMENT_SEARCH_URL : BANK_PAYMENT_SEARCH_URL, form as never as object, 60000, "UTF-8", true);
// const $ = cheerio.load(result);
// const titles = $("div strong")
// .map((_, e) => {
// const titleElement = e as TagElement;
// const text = (titleElement.children[0] as DataNode).data?.trim();
// if (text === undefined) {
// return undefined;
// }
// const res = /(\d+年\d+月)银行代发结果/g.exec(text);
// if (res === null || res[1] === undefined) {
// return undefined;
// }
// if (((titleElement.parentNode?.next?.next as TagElement)?.firstChild as TagElement)?.name !== "table") {
// return undefined;
// }
// return res[1];
// })
// .get()
// .filter((text) => text !== undefined) as string[];
// return $("div table tbody")
// .filter(index => index < titles.length)
// .map((index, e) => {
// const rows = cheerio.load(e)("tr");
// const data = rows.slice(1, rows.length - 1);
// return {
// month: titles[index],
// payment: data.map((_, row) => {
// const columns = cheerio.load(row)("td");
// return {
// department: getCheerioText(columns[1], 0),
// project: getCheerioText(columns[2], 0),
// usage: getCheerioText(columns[3], 0),
// description: getCheerioText(columns[4], 0),
// bank: getCheerioText(columns[5], 0),
// time: getCheerioText(columns[6], 0),
// total: getCheerioText((columns[7] as TagElement).children[0], 0),
// deduction: getCheerioText((columns[8] as TagElement).children[0], 0),
// actual: getCheerioText((columns[9] as TagElement).children[0], 0),
// deposit: getCheerioText((columns[10] as TagElement).children[0], 0),
// cash: getCheerioText((columns[11] as TagElement).children[0], 0),
// } as BankPayment;
// }).get().reverse(),
// };
// })
// .get() as BankPaymentByMonth[];
// },
// MOCK_BANK_PAYMENT,
// );

export const getBankPayment = async (
helper: InfoHelper,
foundation: boolean,
): Promise<BankPaymentByMonth[]> =>
roamingWrapperWithMocks(
helper,
"default",
foundation ? "C1ADD6B60D050B64E0C7B8F195CE89EC" : "2A5182CB3F36E80395FC2091001BDEA6",
async (s) => {
if (s === undefined) {
throw new LibError();
}
const options = cheerio.load(s)("option").map((_, e) => (e as TagElement).attribs.value).get();
if (options.length === 0) {
return [];
}
const form = options.map((o) => `year=${encodeURIComponent(o)}`).join("&");
const result = await uFetch(foundation ? FOUNDATION_BANK_PAYMENT_SEARCH_URL : BANK_PAYMENT_SEARCH_URL, form as never as object, 60000, "UTF-8", true);
const $ = cheerio.load(result);
const titles = $("div strong")
.map((_, e) => {
const titleElement = e as TagElement;
const text = (titleElement.children[0] as DataNode).data?.trim();
if (text === undefined) {
return undefined;
}
const res = /(\d+年\d+月)银行代发结果/g.exec(text);
if (res === null || res[1] === undefined) {
return undefined;
}
if (((titleElement.parentNode?.next?.next as TagElement)?.firstChild as TagElement)?.name !== "table") {
return undefined;
}
return res[1];
})
.get()
.filter((text) => text !== undefined) as string[];
return $("div table tbody")
.filter(index => index < titles.length)
.map((index, e) => {
const rows = cheerio.load(e)("tr");
const data = rows.slice(1, rows.length - 1);
return {
month: titles[index],
payment: data.map((_, row) => {
const columns = cheerio.load(row)("td");
return {
department: getCheerioText(columns[1], 0),
project: getCheerioText(columns[2], 0),
usage: getCheerioText(columns[3], 0),
description: getCheerioText(columns[4], 0),
bank: getCheerioText(columns[5], 0),
time: getCheerioText(columns[6], 0),
total: getCheerioText((columns[7] as TagElement).children[0], 0),
deduction: getCheerioText((columns[8] as TagElement).children[0], 0),
actual: getCheerioText((columns[9] as TagElement).children[0], 0),
deposit: getCheerioText((columns[10] as TagElement).children[0], 0),
cash: getCheerioText((columns[11] as TagElement).children[0], 0),
} as BankPayment;
}).get().reverse(),
};
})
.get() as BankPaymentByMonth[];
},
MOCK_BANK_PAYMENT,
);

export const getBankPaymentParellize = async (
helper: InfoHelper,
foundation: boolean,
loadPartial: boolean = false,
): Promise<BankPaymentByMonth[]> => {
return roamingWrapperWithMocks(
helper,
Expand All @@ -583,52 +584,73 @@ export const getBankPayment = async (
if (options.length === 0) {
return [];
}
const requests = options.map((o) => {
const form = `year=${encodeURIComponent(o)}`;
const loadOptions = (loadPartial ? options.slice(0, 3) : options.map(o => `year=${encodeURIComponent(o)}`));
const jointOptions = [
loadOptions.slice(0, Math.ceil(options.length / 3)).join("&"),
loadOptions
.slice(
Math.floor(options.length / 3),
Math.ceil((2 * options.length) / 3)
)
.join("&"),
loadOptions
.slice(Math.floor((2 * options.length) / 3))
.join("&"),
];
const requests = jointOptions.map((o) => {
const form = `${encodeURIComponent(o)}`;
return uFetch(foundation ? FOUNDATION_BANK_PAYMENT_SEARCH_URL : BANK_PAYMENT_SEARCH_URL, form as never as object, 60000, "UTF-8", true);
});
const results = await Promise.all(requests);
const parsedResults = results.flatMap((result, index) => {
const $ = cheerio.load(result);
const titleElement = $("div strong");
const text = (titleElement[0].children[0] as DataNode).data?.trim();
if (text === undefined) {
return [];
}
const res = /(\d+年\d+月)银行代发结果/g.exec(text);
if (res === null || res[1] === undefined) {
return [];
}
if (((titleElement[0].parentNode?.next?.next as TagElement)?.firstChild as TagElement)?.name !== "table") {
return [];
}

const title = options[index];
const rows = $("div table tbody tr").slice(1, -1);
const payments = rows.map((_, row) => {
const columns = cheerio.load(row)("td");
return {
department: getCheerioText(columns[1], 0),
project: getCheerioText(columns[2], 0),
usage: getCheerioText(columns[3], 0),
description: getCheerioText(columns[4], 0),
bank: getCheerioText(columns[5], 0),
time: getCheerioText(columns[6], 0),
total: getCheerioText((columns[7] as TagElement).children[0], 0),
deduction: getCheerioText((columns[8] as TagElement).children[0], 0),
actual: getCheerioText((columns[9] as TagElement).children[0], 0),
deposit: getCheerioText((columns[10] as TagElement).children[0], 0),
cash: getCheerioText((columns[11] as TagElement).children[0], 0),
} as BankPayment;
}).get().reverse();
return { month: title, payment: payments };
});
const parsedResults = results.map((result) => {
return parseAndFiltBankPayment(result);
}).flatMap((it) => it);
return parsedResults;
},
MOCK_BANK_PAYMENT,
);
};

const parseAndFiltBankPayment = (html: string) => {
const $ = cheerio.load(html);
const titleElements = $("div strong");
return titleElements.map((_, e) => {
const titleElement = e as TagElement;
const text = (titleElement.children[0] as DataNode).data?.trim();
if (text === undefined) {
return undefined;
}
const res = /(\d+年\d+月)银行代发结果/g.exec(text);
if (res === null || res[1] === undefined) {
return undefined;
}
if (((titleElement.parentNode?.next?.next as TagElement)?.firstChild as TagElement)?.name !== "table") {
return undefined;
}
const data = cheerio.load((titleElement.parent?.next?.next as TagElement)?.firstChild as cheerio.AnyNode)("tr").slice(1, -1);
const payment = data.map((_, row) => {
const columns = cheerio.load(row)("td");
return {
department: getCheerioText(columns[1], 0),
project: getCheerioText(columns[2], 0),
usage: getCheerioText(columns[3], 0),
description: getCheerioText(columns[4], 0),
bank: getCheerioText(columns[5], 0),
time: getCheerioText(columns[6], 0),
total: getCheerioText((columns[7] as TagElement).children[0], 0),
deduction: getCheerioText((columns[8] as TagElement).children[0], 0),
actual: getCheerioText((columns[9] as TagElement).children[0], 0),
deposit: getCheerioText((columns[10] as TagElement).children[0], 0),
cash: getCheerioText((columns[11] as TagElement).children[0], 0),
} as BankPayment;
}).get().reverse();
return {
month: res[1],
payment,
};
}).get().filter((it) => it !== undefined) as BankPaymentByMonth[];
};

export const getGraduateIncome = async (
helper: InfoHelper,
begin: string, // YYYYMMDD
Expand Down

0 comments on commit 3ea8a3c

Please sign in to comment.