tweak(tupaiaWeb): RN-1383: Make raw data downloads use one time link

env/api-client.env.example

@@ -3,5 +3,7 @@ DATA_TABLE_API_URL=
 CENTRAL_API_URL=
 ENTITY_API_URL=
 REPORT_API_URL=
-WEB_CONFIG_API_URL=
-API_CLIENT_SALT=
+WEB_CONFIG_API_URL= 
+TUPAIA_WEB_SERVER_API_URL=
+DATATRAK_WEB_SERVER_API_URL=
+ADMIN_PANEL_SERVER_API_URL=

packages/central-server/src/apiV2/DownloadFiles.js

@@ -10,9 +10,8 @@ import {
   getUniqueFileNameParts,
   getDeDuplicatedFileName,
 } from '@tupaia/utils';
-import { S3, S3Client } from '@tupaia/server-utils';
+import { S3, S3Client, getTempDirectory } from '@tupaia/server-utils';
 import { RouteHandler } from './RouteHandler';
-import { getTempDirectory } from '../utilities';
 import { zipMultipleFiles } from './utilities';
 
 export class DownloadFiles extends RouteHandler {

packages/central-server/src/apiV2/export/DownloadHandler.js

@@ -4,13 +4,14 @@
  */
 import fs from 'fs';
 import { respondWithDownload, ValidationError } from '@tupaia/utils';
-import { getExportPathForUser } from './getExportPathForUser';
+import { getExportPathForUser } from '@tupaia/server-utils';
 import { RouteHandler } from '../RouteHandler';
-import { assertAdminPanelAccess } from '../../permissions';
+import { allowNoPermissions } from '../../permissions';
 
 export class DownloadHandler extends RouteHandler {
   async assertUserHasAccess() {
-    await this.assertPermissions(assertAdminPanelAccess);
+    // This is a special case where we don't need to check permissions, as the user must be logged in anyway and the user can't generate an export without specific permissions
+    await this.assertPermissions(allowNoPermissions);
   }
 
   async handleRequest() {

packages/central-server/src/apiV2/export/exportOptionSet/exportOptionSet.js

@@ -5,7 +5,7 @@
 
 import xlsx from 'xlsx';
 import { respondWithDownload, toFilename } from '@tupaia/utils';
-import { getExportPathForUser } from '../getExportPathForUser';
+import { getExportPathForUser } from '@tupaia/server-utils';
 
 const objectToHumanReadableKeyValuePairs = obj => {
   const entries = [];

packages/central-server/src/apiV2/export/exportSurveyResponses/exportResponsesToFile.js

@@ -15,11 +15,11 @@ import {
   truncateString,
   toFilename,
 } from '@tupaia/utils';
+import { getExportPathForUser } from '@tupaia/server-utils';
 import { RECORDS } from '@tupaia/database';
 import { ANSWER_TYPES, NON_DATA_ELEMENT_ANSWER_TYPES } from '../../../database/models/Answer';
 import { findAnswersInSurveyResponse, findQuestionsInSurvey } from '../../../dataAccessors';
 import { hasBESAdminAccess } from '../../../permissions';
-import { getExportPathForUser } from '../getExportPathForUser';
 import { zipMultipleFiles } from '../../utilities';
 
 const FILE_PREFIX = 'survey_response_export';

packages/central-server/src/apiV2/export/exportSurveys/SurveyExporter.js

@@ -5,13 +5,12 @@
 
 import xlsx from 'xlsx';
 import { DatabaseError } from '@tupaia/utils';
-
+import { getExportPathForUser } from '@tupaia/server-utils';
 import { findQuestionsInSurvey } from '../../../dataAccessors';
 import { RowBuilder } from './RowBuilder';
 import { SurveyMetadataConfigCellBuilder } from './cellBuilders';
 import { assertCanExportSurveys } from './assertCanExportSurveys';
 import { assertAnyPermissions, assertBESAdminAccess } from '../../../permissions';
-import { getExportPathForUser } from '../getExportPathForUser';
 
 const FILE_PREFIX = 'survey_export';
 

packages/central-server/src/apiV2/import/index.js

@@ -5,6 +5,7 @@
 
 import express from 'express';
 import multer from 'multer';
+import { getTempDirectory } from '@tupaia/server-utils';
 import { emailAfterTimeout } from '@tupaia/server-boilerplate';
 import { catchAsyncErrors } from '../middleware';
 
@@ -13,7 +14,6 @@ import { importEntities } from './importEntities';
 import { importStriveLabResults } from './importStriveLabResults';
 import { importUsers } from './importUsers';
 import { importSurveyResponses, constructImportEmail } from './importSurveyResponses';
-import { getTempDirectory } from '../../utilities';
 import { importDataElements } from './importDataElements';
 import { importDataElementDataServices } from './importDataElementDataServices';
 import { importUserPermissions } from './importUserPermissions';

packages/central-server/src/apiV2/middleware/multipartJson.js

@@ -4,7 +4,7 @@
  */
 
 import multer from 'multer';
-import { getTempDirectory } from '../../utilities';
+import { getTempDirectory } from '@tupaia/server-utils';
 
 /**
  * Parses a multipartJson request where:
@@ -19,29 +19,31 @@ import { getTempDirectory } from '../../utilities';
  *
  * @param {boolean} [addFilesToBody] If true, will add each File to the body by the given form part name, e.g. myfile: File
  */
-export const multipartJson = (addFilesToBody = true) => async (req, res, next) => {
-  if (req.headers['content-type'].startsWith('multipart/form-data')) {
-    const parserMiddleware = multer({
-      storage: multer.diskStorage({
-        destination: getTempDirectory('uploads'),
-        filename: (req, file, callback) => {
-          callback(null, `${Date.now()}_${file.originalname}`);
-        },
-      }),
-    }).any();
+export const multipartJson =
+  (addFilesToBody = true) =>
+  async (req, res, next) => {
+    if (req.headers['content-type'].startsWith('multipart/form-data')) {
+      const parserMiddleware = multer({
+        storage: multer.diskStorage({
+          destination: getTempDirectory('uploads'),
+          filename: (req, file, callback) => {
+            callback(null, `${Date.now()}_${file.originalname}`);
+          },
+        }),
+      }).any();
 
-    parserMiddleware(req, res, () => {
-      req.body = JSON.parse(req?.body?.payload || '{}');
+      parserMiddleware(req, res, () => {
+        req.body = JSON.parse(req?.body?.payload || '{}');
 
-      if (req.files && addFilesToBody) {
-        for (const file of req.files) {
-          req.body[file.fieldname] = file;
+        if (req.files && addFilesToBody) {
+          for (const file of req.files) {
+            req.body[file.fieldname] = file;
+          }
         }
-      }
 
+        next();
+      });
+    } else {
       next();
-    });
-  } else {
-    next();
-  }
-};
+    }
+  };

packages/central-server/src/utilities/index.js

@@ -4,6 +4,5 @@
  */
 
 export { getApiUrl } from './getApiUrl';
-export { getTempDirectory } from './getTempDirectory';
 export { resourceToRecordType } from './resourceToRecordType';
 export { getStandardisedImageName } from './getStandardisedImageName';

packages/datatrak-web/src/api/mutations/useExportSurveyResponses.ts

@@ -55,6 +55,7 @@ export const useExportSurveyResponses = () => {
             countryCode,
             startDate,
             endDate,
+            platform: 'datatrak',
           },
         });
 

packages/server-utils/src/constructExportEmail.ts

@@ -3,22 +3,11 @@
  * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd
  */
 import path from 'path';
-import fs from 'fs';
+import { requireEnv } from '@tupaia/utils';
 
-enum EmailExportFileModes {
-  ATTACHMENT = 'attachment',
-  DOWNLOAD_LINK = 'downloadLink',
-}
-
-const createDownloadLink = (filePath: string) => {
-  const fileName = path.basename(filePath);
-  return `${process.env.ADMIN_PANEL_SERVER_URL}/v1/export/download/${encodeURIComponent(fileName)}`;
-};
-
-const generateAttachments = async (filePath: string) => {
+const createDownloadLink = (filePath: string, url: string) => {
   const fileName = path.basename(filePath);
-  const buffer = await fs.readFileSync(filePath);
-  return [{ filename: fileName, content: buffer }];
+  return `${url}/export/download/${encodeURIComponent(fileName)}`;
 };
 
 type ResponseBody = {
@@ -28,13 +17,13 @@ type ResponseBody = {
 
 type Req = {
   query: {
-    emailExportFileMode?: EmailExportFileModes;
+    platform?: 'tupaia' | 'adminPanel' | 'datatrak';
   };
 };
 
 export const constructExportEmail = async (responseBody: ResponseBody, req: Req) => {
-  const { emailExportFileMode = EmailExportFileModes.DOWNLOAD_LINK } = req.query;
   const { error, filePath } = responseBody;
+  const { platform } = req.query;
   const subject = 'Your export from Tupaia';
   if (error) {
     return {
@@ -48,18 +37,36 @@ ${error}`,
     throw new Error('No filePath in export response body');
   }
 
-  if (emailExportFileMode === EmailExportFileModes.ATTACHMENT) {
-    return {
-      subject,
-      message: 'Please find your requested export attached to this email.',
-      attachments: await generateAttachments(filePath),
-    };
+  const platformKey = platform || 'adminPanel';
+
+  /**
+   * We don't yet have single sign-on across all platforms, so we need to know which platform the user is on so the correct session is used, from where they requested the export. We also can't use central-server api url for this directly because there needs to be an auth header in the request.
+   */
+  const PLATFORM_SETTINGS = {
+    tupaia: {
+      url: requireEnv('TUPAIA_WEB_SERVER_API_URL'),
+      friendlyName: 'Tupaia',
+    },
+    adminPanel: {
+      url: requireEnv('ADMIN_PANEL_SERVER_API_URL'),
+      friendlyName: 'the Admin Panel',
+    },
+    datatrak: {
+      url: requireEnv('DATATRAK_WEB_SERVER_API_URL'),
+      friendlyName: 'DataTrak',
+    },
+  };
+
+  if (!PLATFORM_SETTINGS[platformKey]) {
+    throw new Error(`No API config found for platform: ${platformKey}`);
   }
 
-  const downloadLink = createDownloadLink(filePath);
+  const { friendlyName, url } = PLATFORM_SETTINGS[platformKey];
+
+  const downloadLink = createDownloadLink(filePath, url);
   return {
     subject,
     message: `Please click this one-time link to download your requested export: ${downloadLink}
-Note that you need to be logged in to the admin panel for it to work, and after clicking it once, you won't be able to download the file again.`,
+  Note that you need to be logged in to ${friendlyName} for it to work, and after clicking it once, you won't be able to download the file again.`,
   };
 };

packages/server-utils/src/getExportPathForUser.ts

@@ -0,0 +1,7 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd
+ */
+import { getTempDirectory } from './getTempDirectory';
+
+export const getExportPathForUser = (userId: string) => getTempDirectory(`exports/${userId}`);

packages/central-server/src/utilities/getTempDirectory.js → packages/server-utils/src/getTempDirectory.ts

@@ -1,13 +1,13 @@
 /**
  * Tupaia
- * Copyright (c) 2017 - 2021 Beyond Essential Systems Pty Ltd
+ * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd
  */
 
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
 
-export function getTempDirectory(name) {
+export function getTempDirectory(name: string) {
   const directoryPath = path.join(os.tmpdir(), 'tupaia', name);
 
   try {

packages/server-utils/src/index.ts

@@ -1,6 +1,13 @@
+/**
+ * Tupaia
+ * Copyright (c) 2017 - 2024 Beyond Essential Systems Pty Ltd
+ */
+
 export { downloadPageAsPDF } from './downloadPageAsPDF';
 export * from './s3';
 export { sendEmail } from './sendEmail';
 export { generateUnsubscribeToken, verifyUnsubscribeToken } from './unsubscribeToken';
 export { configureDotEnv } from './configureDotEnv';
 export { constructExportEmail } from './constructExportEmail';
+export { getTempDirectory } from './getTempDirectory';
+export { getExportPathForUser } from './getExportPathForUser';

packages/tupaia-web-server/src/app/createApp.ts

@@ -99,6 +99,7 @@ export async function createApp(db: TupaiaDatabase = new TupaiaDatabase()) {
     )
     .use('downloadFiles', forwardRequest(CENTRAL_API_URL, { authHandlerProvider }))
     .use('me/countries', forwardRequest(CENTRAL_API_URL, { authHandlerProvider }))
+    .use('export/download/:fileName', forwardRequest(CENTRAL_API_URL, { authHandlerProvider }))
     // Forward everything else to webConfigApi
     .use('dashboards', forwardRequest(WEB_CONFIG_API_URL, { authHandlerProvider }))
     .use('export/chart', forwardRequest(WEB_CONFIG_API_URL, { authHandlerProvider }))

packages/tupaia-web-server/src/routes/export/utils.ts

@@ -2,7 +2,7 @@ const EMAIL_TIMEOUT = 30 * 1000; // 30 seconds
 
 export const EMAIL_TIMEOUT_SETTINGS = {
   respondWithEmailTimeout: EMAIL_TIMEOUT,
-  emailExportFileMode: 'attachment',
+  platform: 'tupaia',
 };
 
 export const handleExportResponse = async (response: {

packages/web-config-server/src/export/ExportSurveyDataHandler.js

@@ -6,14 +6,14 @@
 import xlsx from 'xlsx';
 import fs from 'fs';
 import { getExportDatesString, respondWithDownload } from '@tupaia/utils';
+import { getExportPathForUser } from '@tupaia/server-utils';
 import { requestFromTupaiaConfigServer } from './requestFromTupaiaConfigServer';
 import { USER_SESSION_CONFIG } from '/authSession';
 import { RouteHandler } from '/apiV1/RouteHandler';
 import { ExportSurveyResponsesPermissionsChecker } from '/apiV1/permissions';
 import { formatMatrixDataForExcel } from './excelFormatters/formatMatrixDataForExcel';
 
 const EXPORT_FILE_TITLE = 'survey_response_export';
-const EXPORT_DIRECTORY = 'exports';
 
 export class ExportSurveyDataHandler extends RouteHandler {
   static PermissionsChecker = ExportSurveyResponsesPermissionsChecker;
@@ -108,13 +108,15 @@ export class ExportSurveyDataHandler extends RouteHandler {
       Sheets: sheets,
     };
 
-    if (!(await fs.existsSync(EXPORT_DIRECTORY))) {
-      await fs.mkdirSync(EXPORT_DIRECTORY);
+    const exportDirectory = getExportPathForUser(this.req.user.id);
+
+    if (!(await fs.existsSync(exportDirectory))) {
+      await fs.mkdirSync(exportDirectory);
     }
 
-    const filePath = `${EXPORT_DIRECTORY}/${EXPORT_FILE_TITLE}_${Date.now()}.xlsx`;
+    const filePath = `${exportDirectory}/${EXPORT_FILE_TITLE}_${Date.now()}.xlsx`;
 
     xlsx.writeFile(workbook, filePath);
-    respondWithDownload(this.res, filePath, true);
+    respondWithDownload(this.res, filePath);
   }
 }