Fetch integration data in parallel for update (#1282)

package.json

@@ -61,6 +61,7 @@
 		"linkedom": "^0.16.4",
 		"mdast-util-from-markdown": "^2.0.1",
 		"mdast-util-to-string": "^4.0.0",
+		"p-limit": "^6.1.0",
 		"puppeteer": "^21.6.0",
 		"slugify": "^1.6.6",
 		"tiny-glob": "^0.2.9",

scripts/npm.mjs

@@ -1,14 +1,19 @@
 import { format, subDays } from 'date-fns';
+import pLimit from 'p-limit';
 
-async function fetchJson(url) {
-	const res = await fetch(url);
+const fetchLimit = pLimit(10);
 
-	if (!res.ok) {
-		console.error(`[${url}] ${res.status} ${res.statusText}`);
-		throw new Error();
-	}
+function fetchJson(url) {
+	return fetchLimit(async () => {
+		const res = await fetch(url);
 
-	return await res.json();
+		if (!res.ok) {
+			console.error(`[${url}] ${res.status} ${res.statusText}`);
+			throw new Error();
+		}
+
+		return await res.json();
+	})
 }
 
 const API_BASE_URL = 'https://api.npmjs.org/';

scripts/update-integrations.mjs

@@ -118,7 +118,7 @@ async function unsafeUpdateAllIntegrations() {
 	const deprecatedIntegrations = [];
 
 	// loop through all integrations already published to the catalog
-	for (const entry of entries) {
+	await Promise.all(entries.map(async entry => {
 		const { data } = matter.read(entry);
 		existingIntegrations.add(data.name);
 
@@ -144,14 +144,14 @@ async function unsafeUpdateAllIntegrations() {
 ${frontmatter}---\n`,
 			);
 		}
-	}
+	}));
 
 	// find new integrations that haven't been published yet
 	const newIntegrations = Array.from(searchResults.keys()).filter(
 		(pkg) => !existingIntegrations.has(pkg),
 	);
 
-	for (const entry of newIntegrations) {
+	await Promise.all(newIntegrations.map(async (entry) => {
 		const details = await fetchWithOverrides(entry);
 
 		const frontmatter = yaml.stringify(details);
@@ -167,7 +167,7 @@ ${frontmatter}---\n`,
 			`---
 ${frontmatter}---\n`,
 		);
-	}
+	}));
 
 	updateLastModified();