From f84315d8f5701c673b70603cc3c02c805637e9d2 Mon Sep 17 00:00:00 2001 From: Lachezar Hristov Date: Wed, 9 Oct 2024 13:09:20 +0200 Subject: [PATCH] Allow for larger forms to create csv exports --- .../scripts/generate-csv-data-set/generate-csv-data-set.js | 3 ++- server/src/scripts/generate-csv/bin.js | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/scripts/generate-csv-data-set/generate-csv-data-set.js b/server/src/scripts/generate-csv-data-set/generate-csv-data-set.js index b6de2ff816..4aa14f16c9 100755 --- a/server/src/scripts/generate-csv-data-set/generate-csv-data-set.js +++ b/server/src/scripts/generate-csv-data-set/generate-csv-data-set.js @@ -43,7 +43,8 @@ function generateCsv(dbName, formId, outputPath, year = '*', month = '*', csvTem } cmd = `${cmd} ${csvTemplate ? `"${csvTemplate.headers.join(',')}"` : ''}` log.debug("generate-csv: " + cmd) - exec(cmd).then(status => { + const maxBuffer = 1024 * 1024 * 100; + exec(cmd, { maxBuffer }).then(status => { resolve(status) }).catch(error => { log.error("Error when exec-ing generate-csv: " + error) diff --git a/server/src/scripts/generate-csv/bin.js b/server/src/scripts/generate-csv/bin.js index eb2e1ae966..5f05bd0b9c 100755 --- a/server/src/scripts/generate-csv/bin.js +++ b/server/src/scripts/generate-csv/bin.js @@ -78,7 +78,9 @@ async function go(state) { // Run batches. while (state.complete === false) { log.debug(`Run batch at skip of ${state.skip} at statePath: ${state.statePath}`) - const response = await exec(`./batch.js '${state.statePath}'`) + const maxBuffer = 1024 * 1024 * 100; // 100MB + const response = await exec(`./batch.js '${state.statePath}'`, { maxBuffer }); + if (response.stderr) console.error(response.stderr) if (process.env.NODE_ENV === 'development') console.log(response) state = JSON.parse(await readFile(state.statePath))