diff --git a/CHANGELOG.md b/CHANGELOG.md index 0456c4e0a3..21ce7e7ddc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,47 @@ # What's new +## v3.31.0 + +__New Features__ + +- **Audio and Visual Feedback**: A new Prompt Box widget available in form authoring allows a form designer to add audio and visual feedback connected to Radio Block widgets. This feature provides a toolset for creating self-guided assessments. See the example in [tangy-forms](https://github.com/Tangerine-Community/tangy-form/blob/master/CHANGELOG.md#v4430). [#3473](https://github.com/Tangerine-Community/Tangerine/issues/3473) + +- Client Login Screen Custom HTML: A new app-config.json setting, `customLoginMarkup`, allows for custom HTML to be added to the login screen. This feature is useful for adding custom branding or additional information to the login screen. As an example: +```json +"customLoginMarkup": "
logo
" +``` + +- **Improved Data Management**: + * Data Managers now have access to a full workflow to review, edit, and verify data in the Tangerine web server. The Data Manager can click on a record and enter a new screen that allows them to perform actions align with a data collection supervision process. + * Searching has been improved to allow seaqrching for a specific ID in the imported data. This feature is useful for finding specific records synced to the server when reviewing or editing completed form responses. [#3681](https://github.com/Tangerine-Community/Tangerine/issues/3681) + +__Fixes__ +- Client Search Service: exclude archived cases from recent activity +- Media library cannot upload photos [#3583](https://github.com/Tangerine-Community/Tangerine/issues/3583) +- User Profile Import: The process of importing an existing device user now allows for retries and an asynchronous process to download existing records. This fixes an issue cause by timeouts when trying to import a user with a large number of records. [#3696](https://github.com/Tangerine-Community/Tangerine/issues/3696) +- When `T_ONLY_PROCESS_THESE_GROUPS` has a list of one or more groups, running `reporting-cache-clear` will only process the groups in the list + +__Tangerine Teach__ + +- Add toggle in Attendence Check for 'late'. A teacher can click through the status of 'present', 'late', or 'absent' for each student. +- Use `studentRegistrationFields` to control showing name and surname of student in the student dashboard + +__Libs and Dependencies__ +- Bump version of `tangy-form` to v4.31.1 and `tangy-form-editor` to v7.18.0 for the new Prompt Box widget +- Bump version of `tangy-form` to v4.45.1 for disabling of `tangy-gps` in server edits + +__Server upgrade instructions__ + +See the [Server Upgrade Insturctions](https://docs.tangerinecentral.org/system-administrator/upgrade-instructions). + +*Special Instructions for this release:* + +Once the Tangerine and CouchDB are running, run the upgrade script for v3.31.0: + +`docker exec -it tangerine /tangerine/server/src/upgrade/v3.31.0.js` + + + ## v3.30.2 __New Features__ diff --git a/client/package.json b/client/package.json index f2093ac35f..a87023f962 100644 --- a/client/package.json +++ b/client/package.json @@ -75,7 +75,7 @@ "rxjs-compat": "^6.5.5", "sanitize-filename-ts": "^1.0.2", "spark-md5": "^3.0.2", - "tangy-form": "4.42.0", + "tangy-form": "^4.43.2", "translation-web-component": "1.1.1", "tslib": "^1.10.0", "underscore": "^1.9.1", diff --git a/client/src/app/class/_services/dashboard.service.ts b/client/src/app/class/_services/dashboard.service.ts index 98203bc29a..87995e4386 100644 --- a/client/src/app/class/_services/dashboard.service.ts +++ b/client/src/app/class/_services/dashboard.service.ts @@ -1043,6 +1043,37 @@ export class DashboardService { return gradeInput?.value } + async getAllStudentResults(students, studentsResponses, curriculumFormsList, curriculum) { + const allStudentResults = []; + + const appConfig = await this.appConfigService.getAppConfig(); + const studentRegistrationFields = appConfig.teachProperties?.studentRegistrationFields || [] + + students.forEach((student) => { + const studentResult = {}; + + studentResult['id'] = student.id; + studentRegistrationFields.forEach((field) => { + studentResult[field] = this.getValue(field, student.doc) + }) + studentResult['forms'] = {}; + curriculumFormsList.forEach((form) => { + const formResult = {}; + formResult['formId'] = form.id; + formResult['curriculum'] = curriculum.name; + formResult['title'] = form.title; + formResult['src'] = form.src; + if (studentsResponses[student.id]) { + formResult['response'] = studentsResponses[student.id][form.id]; + } + studentResult['forms'][form.id] = formResult; + }); + allStudentResults.push(studentResult); + }); + + return allStudentResults; + } + /** * Get the attendance list for the class, including any students who have not yet had attendance checked. If the savedAttendanceList is passed in, then * populate the student from that doc by matching student.id. @@ -1050,10 +1081,9 @@ export class DashboardService { * @param savedList */ async getAttendanceList(students, savedList, curriculum) { - // const curriculumFormHtml = await this.getCurriculaForms(curriculum.name); - // const curriculumFormsList = await this.classUtils.createCurriculumFormsList(curriculumFormHtml); const appConfig = await this.appConfigService.getAppConfig(); const studentRegistrationFields = appConfig.teachProperties?.studentRegistrationFields || [] + const list = [] for (const student of students) { let studentResult @@ -1062,13 +1092,6 @@ export class DashboardService { studentResult = savedList.find(studentDoc => studentDoc.id === studentId) } if (studentResult) { - // migration. - if (!studentResult.student_surname) { - studentResult.student_surname = studentResult.surname - } - if (!studentResult.student_name) { - studentResult.student_name = studentResult.name - } list.push(studentResult) } else { studentResult = {} @@ -1077,22 +1100,15 @@ export class DashboardService { studentRegistrationFields.forEach((field) => { studentResult[field] = this.getValue(field, student.doc) }) - // const student_name = this.getValue('student_name', student.doc) - // const student_surname = this.getValue('student_surname', student.doc) - // const phone = this.getValue('phone', student.doc); - // const classId = this.getValue('classId', student.doc) - - // studentResult['name'] = student_name - // studentResult['surname'] = student_surname - // studentResult['phone'] = phone - // studentResult['classId'] = classId studentResult['absent'] = null + if (appConfig.teachProperties?.showLateAttendanceOption) { + studentResult['late'] = null + } list.push(studentResult) } } return list - // await this.populateFeedback(curriculumId); } /** diff --git a/client/src/app/class/attendance/attendance-check/attendance-check.component.html b/client/src/app/class/attendance/attendance-check/attendance-check.component.html index 9177b2ae37..dd6ca0d95f 100644 --- a/client/src/app/class/attendance/attendance-check/attendance-check.component.html +++ b/client/src/app/class/attendance/attendance-check/attendance-check.component.html @@ -13,14 +13,28 @@ {{element["student_name"]}} {{element["student_surname"]}} - + + + + + + + + + + - + + close + + + + remove - + check diff --git a/client/src/app/class/attendance/attendance-check/attendance-check.component.ts b/client/src/app/class/attendance/attendance-check/attendance-check.component.ts index 90c82f7395..16d0f25b72 100644 --- a/client/src/app/class/attendance/attendance-check/attendance-check.component.ts +++ b/client/src/app/class/attendance/attendance-check/attendance-check.component.ts @@ -9,6 +9,7 @@ import {UserService} from "../../../shared/_services/user.service"; import {FormMetadata} from "../../form-metadata"; import {ClassFormService} from "../../_services/class-form.service"; import { TangySnackbarService } from 'src/app/shared/_services/tangy-snackbar.service'; +import { AppConfigService } from 'src/app/shared/_services/app-config.service'; @Component({ selector: 'app-attendance-check', @@ -40,16 +41,22 @@ export class AttendanceCheckComponent implements OnInit { curriculum:any ignoreCurriculumsForTracking: boolean = false reportLocaltime: string; + showLateAttendanceOption: boolean = false; constructor( private dashboardService: DashboardService, private variableService : VariableService, private router: Router, private classFormService: ClassFormService, - private tangySnackbarService: TangySnackbarService + private tangySnackbarService: TangySnackbarService, + private appConfigService: AppConfigService ) { } async ngOnInit(): Promise { + + const appConfig = await this.appConfigService.getAppConfig() + this.showLateAttendanceOption = appConfig.teachProperties.showLateAttendanceOption || this.showLateAttendanceOption + let classIndex await this.classFormService.initialize(); this.getValue = this.dashboardService.getValue @@ -133,18 +140,33 @@ export class AttendanceCheckComponent implements OnInit { } - async toggleAttendance(student) { - student.absent = !student.absent - await this.saveStudentAttendance() - } - - async toggleMood(mood, student) { - student.mood = mood - if (!student.absent) { - await this.saveStudentAttendance() + async toggleAttendance(currentStatus, student) { + if (this.showLateAttendanceOption) { + if (currentStatus == 'present') { + // moving from present status to late status + student.absent = false + student.late = true + } else if (currentStatus == 'late') { + // moving from late status to absent status + student.absent = true + student.late = false + } else { + // moving from absent status to present status + student.absent = false + student.late = false + } + } else { + if (currentStatus == 'present') { + // moving from present status to absent status + student.absent = true + } else { + // moving from absent status to present status + student.absent = false + } } - } + await this.saveStudentAttendance() + } private async saveStudentAttendance() { // save allStudentResults diff --git a/client/src/app/class/attendance/attendance-dashboard/attendance-dashboard.component.css b/client/src/app/class/attendance/attendance-dashboard/attendance-dashboard.component.css index f328b8da65..20eeac4954 100644 --- a/client/src/app/class/attendance/attendance-dashboard/attendance-dashboard.component.css +++ b/client/src/app/class/attendance/attendance-dashboard/attendance-dashboard.component.css @@ -199,6 +199,9 @@ mat-card-title { .mat-chip.mat-standard-chip.mat-chip-selected.mat-primary.red { background-color: red; } +.mat-chip.mat-standard-chip.mat-chip-selected.mat-primary.orange { + background-color: orange; +} .gray { background-color: gray; diff --git a/client/src/app/class/attendance/attendance-dashboard/attendance-dashboard.component.ts b/client/src/app/class/attendance/attendance-dashboard/attendance-dashboard.component.ts index 385d3a2c68..be5e03cceb 100644 --- a/client/src/app/class/attendance/attendance-dashboard/attendance-dashboard.component.ts +++ b/client/src/app/class/attendance/attendance-dashboard/attendance-dashboard.component.ts @@ -112,7 +112,7 @@ export class AttendanceDashboardComponent implements OnInit { for (let i = 0; i < this.currArray.length; i++) { const curriculum = this.currArray[i]; let curriculumLabel = curriculum?.label - const reports = await this.dashboardService.searchDocs('scores', currentClass, null, null, curriculumLabel, randomId, true) + const reports = await this.dashboardService.searchDocs('scores', currentClass, '*', null, curriculumLabel, randomId, true) reports.forEach((report) => { report.doc.curriculum = curriculum scoreReports.push(report.doc) diff --git a/client/src/app/class/dashboard/dashboard.component.css b/client/src/app/class/dashboard/dashboard.component.css index 1e742f709b..afa6fc323b 100644 --- a/client/src/app/class/dashboard/dashboard.component.css +++ b/client/src/app/class/dashboard/dashboard.component.css @@ -194,6 +194,10 @@ mat-card-title { background-color: red; } +.mat-chip.mat-standard-chip.mat-chip-selected.mat-primary.orange { + background-color: orange; +} + .gray { background-color: gray; } diff --git a/client/src/app/class/dashboard/dashboard.component.html b/client/src/app/class/dashboard/dashboard.component.html index 5053018c81..481be252f3 100644 --- a/client/src/app/class/dashboard/dashboard.component.html +++ b/client/src/app/class/dashboard/dashboard.component.html @@ -58,7 +58,7 @@ {{'Completed?'|translate}} - {{element["name"]}} + {{element["student_name"]}} {{element["student_surname"]}} { - const studentResults = {}; - const student_name = this.getValue('student_name', student.doc) - const classId = this.getValue('classId', student.doc) - studentResults['id'] = student.id; - studentResults['name'] = student_name - studentResults['classId'] = classId - // studentResults["forms"] = []; - studentResults['forms'] = {}; - // for (const form of this.curriculumForms) { - this.curriculumFormsList.forEach((form) => { - const formResult = {}; - formResult['formId'] = form.id; - formResult['curriculum'] = this.curriculum.name; - formResult['title'] = form.title; - formResult['src'] = form.src; - if (this.studentsResponses[student.id]) { - formResult['response'] = this.studentsResponses[student.id][form.id]; - } - // studentResults["forms"].push(formResult) - studentResults['forms'][form.id] = formResult; - }); - allStudentResults.push(studentResults); - }); - this.allStudentResults = allStudentResults; - // await this.populateFeedback(curriculumId); + this.allStudentResults = await this.dashboardService.getAllStudentResults(this.students, this.studentsResponses, this.curriculumFormsList, this.curriculum); } // Triggered by dropdown selection in UI. diff --git a/client/src/app/class/feedback.ts b/client/src/app/class/feedback.ts index a915e5f1d4..40e2e8d4b8 100644 --- a/client/src/app/class/feedback.ts +++ b/client/src/app/class/feedback.ts @@ -7,6 +7,7 @@ export class Feedback { skill:string; assignment:string; message:string; + customJSCode: string; messageTruncated: string; // for listing calculatedScore:string; percentileRange: string; diff --git a/client/src/app/class/reports/attendance/attendance.component.css b/client/src/app/class/reports/attendance/attendance.component.css index fe7f437068..dbb152ecc5 100644 --- a/client/src/app/class/reports/attendance/attendance.component.css +++ b/client/src/app/class/reports/attendance/attendance.component.css @@ -60,6 +60,11 @@ .green { background-color: green; } + +.orange { + background-color: orange; +} + .mat-chip.mat-standard-chip.mat-primary { background-color: green; } @@ -76,6 +81,10 @@ background-color: red; } +.mat-chip.mat-standard-chip.mat-chip-selected.mat-primary.orange { + background-color: orange; +} + .tangy-class-card-content-container { margin-left: 3%; /*width: 90%;*/ diff --git a/client/src/app/class/reports/attendance/attendance.component.ts b/client/src/app/class/reports/attendance/attendance.component.ts index f2eb5dd0f4..e66c0343ae 100644 --- a/client/src/app/class/reports/attendance/attendance.component.ts +++ b/client/src/app/class/reports/attendance/attendance.component.ts @@ -117,6 +117,8 @@ export class AttendanceComponent implements OnInit { unitDate.endDate = endDate }) + this.reportLocaltime = DateTime.now().toLocaleString(DateTime.DATE_FULL) + // maybe make this relative to the selected date for the single day report table this.rangeStartDate = DateTime.now().minus({months: 1}).toJSDate(); this.rangeEndDate = DateTime.now().toJSDate(); @@ -205,17 +207,7 @@ export class AttendanceComponent implements OnInit { scoreReports.push(report.doc) }) } - const currentScoreReport = scoreReports[scoreReports.length - 1] - - if (currentAttendanceReport?.timestamp) { - const timestampFormatted = DateTime.fromMillis(currentAttendanceReport?.timestamp) - // DATE_MED - this.reportLocaltime = timestampFormatted.toLocaleString(DateTime.DATE_FULL) - } else { - this.reportLocaltime = DateTime.now().toLocaleString(DateTime.DATE_FULL) - } - - let scoreReport = currentScoreReport + let scoreReport = scoreReports[scoreReports.length - 1] if (startDate && endDate) { const selectedAttendanceReports = attendanceReports @@ -332,11 +324,6 @@ export class AttendanceComponent implements OnInit { selectStudentDetails(student) { student.ignoreCurriculumsForTracking = this.ignoreCurriculumsForTracking - const studentId = student.id; - const classId = student.classId; - // this.router.navigate(['student-details'], { queryParams: - // { studentId: studentId, classId: classId } - // }); this._bottomSheet.open(StudentDetailsComponent, { data: { student: student, @@ -352,12 +339,26 @@ export class AttendanceComponent implements OnInit { this.setBackButton(updatedIndex) this.currentIndex = updatedIndex this.attendanceReport = await this.generateSummaryReport(this.currArray, this.curriculum, this.selectedClass, this.classId, this.currentIndex, null, null); + + if ( this.attendanceReport?.timestamp) { + const timestampFormatted = DateTime.fromMillis( this.attendanceReport?.timestamp) + this.reportLocaltime = timestampFormatted.toLocaleString(DateTime.DATE_FULL) + } else { + this.reportLocaltime = DateTime.now().toLocaleString(DateTime.DATE_FULL) + } } async goForward() { const updatedIndex = this.currentIndex - 1 this.setForwardButton(updatedIndex); this.currentIndex = updatedIndex this.attendanceReport = await this.generateSummaryReport(this.currArray, this.curriculum, this.selectedClass, this.classId, this.currentIndex, null, null); + + if ( this.attendanceReport?.timestamp) { + const timestampFormatted = DateTime.fromMillis( this.attendanceReport?.timestamp) + this.reportLocaltime = timestampFormatted.toLocaleString(DateTime.DATE_FULL) + } else { + this.reportLocaltime = DateTime.now().toLocaleString(DateTime.DATE_FULL) + } } private setBackButton(updatedIndex) { diff --git a/client/src/app/class/reports/student-grouping-report/student-grouping-report.component.html b/client/src/app/class/reports/student-grouping-report/student-grouping-report.component.html index c54f03e267..9d9131a420 100644 --- a/client/src/app/class/reports/student-grouping-report/student-grouping-report.component.html +++ b/client/src/app/class/reports/student-grouping-report/student-grouping-report.component.html @@ -59,13 +59,13 @@

{{'Student'|translate}} - {{element["name"]}} + {{element["name"]}} {{'Score'|translate}} - + {{element.customScore?element.customScore:tNumber(element.score)}} / @@ -77,7 +77,7 @@

{{'Percentile'|translate}} - + {{tNumber(element.scorePercentageCorrect)}} % @@ -85,7 +85,7 @@

{{'Status'|translate}} - + {{element.status}} diff --git a/client/src/app/class/reports/student-grouping-report/student-grouping-report.component.ts b/client/src/app/class/reports/student-grouping-report/student-grouping-report.component.ts index e1372c4688..88cc172540 100644 --- a/client/src/app/class/reports/student-grouping-report/student-grouping-report.component.ts +++ b/client/src/app/class/reports/student-grouping-report/student-grouping-report.component.ts @@ -154,7 +154,7 @@ export class StudentGroupingReportComponent implements OnInit { } } - async getFeedbackForPercentile(percentile, curriculumId, itemId, name) { + async getFeedbackForPercentile(percentile, curriculumId, itemId, name, studentId) { // console.log("Get feedback for " + JSON.stringify(element)) const feedback: Feedback = await this.dashboardService.getFeedback(percentile, curriculumId, itemId); if (feedback) { @@ -163,7 +163,7 @@ export class StudentGroupingReportComponent implements OnInit { } // this.checkFeedbackMessagePosition = true; const dialogRef = this.dialog.open(FeedbackDialog, { - data: {classGroupReport: this.classGroupReport, name: name}, + data: {classGroupReport: this.classGroupReport, name: name, studentId}, width: '95vw', maxWidth: '95vw', }); @@ -209,19 +209,24 @@ export class StudentGroupingReportComponent implements OnInit { export interface DialogData { classGroupReport: ClassGroupingReport; name: string; + studentId } @Component({ selector: 'feedback-dialog', templateUrl: 'feedback-dialog.html', }) -export class FeedbackDialog { +export class FeedbackDialog implements OnInit { constructor( public dialogRef: MatDialogRef, @Inject(MAT_DIALOG_DATA) public data: DialogData, ) { } - + async ngOnInit(){ + if(this.data.classGroupReport?.feedback?.customJSCode){ + eval(this.data.classGroupReport?.feedback?.customJSCode) + } + } tNumber(fragment) { return tNumber(fragment) } diff --git a/client/src/app/core/auth/login/login.component.html b/client/src/app/core/auth/login/login.component.html index 900ed25139..21084a85e2 100755 --- a/client/src/app/core/auth/login/login.component.html +++ b/client/src/app/core/auth/login/login.component.html @@ -82,3 +82,4 @@ + \ No newline at end of file diff --git a/client/src/app/core/auth/login/login.component.ts b/client/src/app/core/auth/login/login.component.ts index b7b8341a54..fa0ff24068 100755 --- a/client/src/app/core/auth/login/login.component.ts +++ b/client/src/app/core/auth/login/login.component.ts @@ -5,7 +5,7 @@ import { DeviceService } from './../../../device/services/device.service'; import {from as observableFrom, Observable } from 'rxjs'; -import { Component, OnInit } from '@angular/core'; +import { Component, ElementRef, OnInit, ViewChild } from '@angular/core'; import { ActivatedRoute, Router } from '@angular/router'; import { AppConfigService } from '../../../shared/_services/app-config.service'; @@ -33,6 +33,7 @@ export class LoginComponent implements OnInit { hidePassword = true passwordPolicy: string passwordRecipe: string + @ViewChild('customLoginMarkup', {static: true}) customLoginMarkup: ElementRef; constructor( private route: ActivatedRoute, private router: Router, @@ -59,6 +60,7 @@ export class LoginComponent implements OnInit { if (this.userService.isLoggedIn()) { this.router.navigate([this.returnUrl]); } + this.customLoginMarkup.nativeElement.innerHTML = appConfig.customLoginMarkup || ''; } diff --git a/client/src/app/core/export-data/export-data/export-data.component.ts b/client/src/app/core/export-data/export-data/export-data.component.ts index 870ad77e75..94157f4750 100644 --- a/client/src/app/core/export-data/export-data/export-data.component.ts +++ b/client/src/app/core/export-data/export-data/export-data.component.ts @@ -275,11 +275,12 @@ export class ExportDataComponent implements OnInit { const stream = new window['Memorystream'] let data = ''; stream.on('data', function (chunk) { - data += chunk.toString(); + data += chunk.toString() +',';// Add comma after each item - will be useful in creating a valid JSON object }); await db.dump(stream) console.log('Successfully exported : ' + dbName); this.statusMessage += `

${_TRANSLATE('Successfully exported database ')} ${dbName}

` + data = `[${data.replace(/,([^,]*)$/, '$1')}]`//Make a valid JSON string - Wrap the items in [] and remove trailing comma const file = new Blob([data], {type: 'application/json'}); this.downloadData(file, fileName, 'application/json'); this.hideExportButton = false diff --git a/client/src/app/shared/_classes/app-config.class.ts b/client/src/app/shared/_classes/app-config.class.ts index 9694b2a680..2479a5589b 100644 --- a/client/src/app/shared/_classes/app-config.class.ts +++ b/client/src/app/shared/_classes/app-config.class.ts @@ -10,7 +10,8 @@ export class AppConfig { // Tangerine Case Management: 'case-home' // Tangerine Teach: 'dashboard' homeUrl = "case-management" - + // customLoginMarkup Custom Markup to include on the login page + customLoginMarkup: string // // i18n configuration. // @@ -150,7 +151,8 @@ export class AppConfig { behaviorSecondaryThreshold: 80, useAttendanceFeature: false, showAttendanceCalendar: false, - studentRegistrationFields:[] + studentRegistrationFields:[], + showLateAttendanceOption: false } // diff --git a/client/src/app/shared/_services/search.service.ts b/client/src/app/shared/_services/search.service.ts index b363eb220b..14a53927ca 100644 --- a/client/src/app/shared/_services/search.service.ts +++ b/client/src/app/shared/_services/search.service.ts @@ -44,7 +44,7 @@ export class SearchService { await createSearchIndex(db, formsInfo, customSearchJs) } - async search(username:string, phrase:string, limit = 50, skip = 0):Promise> { + async search(username:string, phrase:string, limit = 50, skip = 0, excludeArchived = true):Promise> { const db = await this.userService.getUserDatabase(username) let result:any = {} let activity = [] @@ -53,13 +53,18 @@ export class SearchService { } // Only show activity if they have enough activity to fill a page. if (phrase === '' && activity.length >= 11) { - const page = activity.slice(skip, skip + limit) + let page = activity.slice(skip, skip + limit) result = await db.allDocs( { keys: page, include_docs: true } ) + if (excludeArchived) { + // Filter out archived + page = page.filter(id => !result.rows.find(row => row.id === id).doc.archived); + } + // Sort it because the order of the docs returned is not guaranteed by the order of the keys parameter. result.rows = page.map(id => result.rows.find(row => row.id === id)) } else { diff --git a/client/src/app/user-profile/import-user-profile/import-user-profile.component.css b/client/src/app/user-profile/import-user-profile/import-user-profile.component.css index 3383624c0b..31720ef5d3 100644 --- a/client/src/app/user-profile/import-user-profile/import-user-profile.component.css +++ b/client/src/app/user-profile/import-user-profile/import-user-profile.component.css @@ -5,4 +5,7 @@ } paper-input { margin: 15px; -} \ No newline at end of file +} +#err { + color: red; + } diff --git a/client/src/app/user-profile/import-user-profile/import-user-profile.component.html b/client/src/app/user-profile/import-user-profile/import-user-profile.component.html index 71ad6c9154..0841d00108 100644 --- a/client/src/app/user-profile/import-user-profile/import-user-profile.component.html +++ b/client/src/app/user-profile/import-user-profile/import-user-profile.component.html @@ -3,5 +3,11 @@ {{'submit'|translate}}

- {{'Syncing'|translate}}... + {{'Synced'|translate}} {{processedDocs}} {{'of'|translate}} {{totalDocs}} +

+

+ {{'Record with import code'|translate}} {{shortCode}} {{'not found.'|translate}} +

+

+ {{'Import not successful. Click Submit to retry.' |translate}}

diff --git a/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts b/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts index a88533c083..39a07a0787 100644 --- a/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts +++ b/client/src/app/user-profile/import-user-profile/import-user-profile.component.ts @@ -2,10 +2,10 @@ import { Component, ViewChild, ElementRef, AfterContentInit } from '@angular/cor import { HttpClient } from '@angular/common/http'; import { UserService } from '../../shared/_services/user.service'; import { Router } from '@angular/router'; -import PouchDB from 'pouchdb'; import { AppConfigService } from 'src/app/shared/_services/app-config.service'; import { AppConfig } from 'src/app/shared/_classes/app-config.class'; import { _TRANSLATE } from 'src/app/shared/translation-marker'; +import { VariableService } from 'src/app/shared/_services/variable.service'; @Component({ @@ -17,42 +17,72 @@ export class ImportUserProfileComponent implements AfterContentInit { STATE_SYNCING = 'STATE_SYNCING' STATE_INPUT = 'STATE_INPUT' + STATE_ERROR = 'STATE_ERROR' + STATE_NOT_FOUND ='STATE_NOT_FOUND' appConfig: AppConfig state = this.STATE_INPUT docs; + totalDocs; + processedDocs = 0; + userAccount; + db; + shortCode @ViewChild('userShortCode', {static: true}) userShortCodeInput: ElementRef; constructor( private router: Router, private http: HttpClient, private userService: UserService, - private appConfigService: AppConfigService - ) { } + private appConfigService: AppConfigService, + private variableService: VariableService + ) { } ngAfterContentInit() { } async onSubmit() { const username = this.userService.getCurrentUser() - const db = await this.userService.getUserDatabase(this.userService.getCurrentUser()) - const userAccount = await this.userService.getUserAccount(this.userService.getCurrentUser()) + this.db = await this.userService.getUserDatabase(username) + this.userAccount = await this.userService.getUserAccount(username) try { - const profileToReplace = await db.get(userAccount.userUUID) - await db.remove(profileToReplace) + const profileToReplace = await this.db.get(this.userAccount.userUUID) + await this.db.remove(profileToReplace) } catch(e) { // It's ok if this fails. It's probably because they are trying again and the profile has already been deleted. } - this.state = this.STATE_SYNCING - this.appConfig = await this.appConfigService.getAppConfig() - const shortCode = this.userShortCodeInput.nativeElement.value - this.docs = await this.http.get(`${this.appConfig.serverUrl}api/${this.appConfig.groupId}/responsesByUserProfileShortCode/${shortCode}`).toPromise() - const newUserProfile = this.docs.find(doc => doc.form && doc.form.id === 'user-profile') - await this.userService.saveUserAccount({...userAccount, userUUID: newUserProfile._id, initialProfileComplete: true}) - for (let doc of this.docs) { - delete doc._rev - await db.put(doc) - } - this.router.navigate([`/${this.appConfig.homeUrl}`] ); + await this.startSyncing() + } + + async startSyncing(){ + try { + this.appConfig = await this.appConfigService.getAppConfig() + this.shortCode = this.userShortCodeInput.nativeElement.value; + let existingUserProfile = await this.http.get(`${this.appConfig.serverUrl}api/${this.appConfig.groupId}/userProfileByShortCode/${this.shortCode}`).toPromise() + if(!!existingUserProfile){ + const username = this.userService.getCurrentUser() + this.state = this.STATE_SYNCING + await this.userService.saveUserAccount({ ...this.userAccount, userUUID: existingUserProfile['_id'], initialProfileComplete: true }) + this.totalDocs = (await this.http.get(`${this.appConfig.serverUrl}api/${this.appConfig.groupId}/responsesByUserProfileShortCode/${this.shortCode}/?totalRows=true`).toPromise())['totalDocs'] + const docsToQuery = 1000; + let previousProcessedDocs = await this.variableService.get(`${username}-processedDocs`) + let processedDocs = parseInt(previousProcessedDocs) || 0; + while (processedDocs < this.totalDocs) { + this.docs = await this.http.get(`${this.appConfig.serverUrl}api/${this.appConfig.groupId}/responsesByUserProfileShortCode/${this.shortCode}/${docsToQuery}/${processedDocs}`).toPromise() + for (let doc of this.docs) { + delete doc._rev + await this.db.put(doc) + } + processedDocs += this.docs.length; + this.processedDocs = processedDocs + await this.variableService.set(`${username}-processedDocs`, String(processedDocs)) + } + } else { + this.state = this.STATE_NOT_FOUND + } + this.router.navigate([`/${this.appConfig.homeUrl}`] ); + } catch (error) { + this.state = this.STATE_ERROR + } } -} +} \ No newline at end of file diff --git a/client/src/polyfills.ts b/client/src/polyfills.ts index 6e8bf057b8..2e45b6d7b8 100644 --- a/client/src/polyfills.ts +++ b/client/src/polyfills.ts @@ -48,6 +48,7 @@ import 'tangy-form/input/tangy-signature.js'; import 'tangy-form/input/tangy-toggle.js'; import 'tangy-form/input/tangy-radio-blocks.js'; import 'tangy-form/input/tangy-video-capture.js'; +import 'tangy-form/input/tangy-prompt-box.js'; // import 'tangy-form/input/tangy-scan-image.js'; diff --git a/config.defaults.sh b/config.defaults.sh index f2ac9c8ebd..4e4d551031 100755 --- a/config.defaults.sh +++ b/config.defaults.sh @@ -28,6 +28,8 @@ T_MYSQL_CONTAINER_NAME="mysql" T_MYSQL_USER="admin" T_MYSQL_PASSWORD="password" T_MYSQL_PHPMYADMIN="FALSE" +# Enter "true" if using a mysql container instead of an external database service such as AWS RDS. This will launch a mysql container. +T_USE_MYSQL_CONTAINER="" # # Optional @@ -62,12 +64,12 @@ T_REPORTING_DELAY="300000" # Number of change docs from the Couchdb changes feed queried by reporting-worker (i.e. use as the limit parameter) T_LIMIT_NUMBER_OF_CHANGES=200 -# Limit processing to certain group dbs. +# Limit processing to certain group dbs. Cache clear and batching reporting outputs will only run on the groups specified below. +# If empty, all groups will be processed. +# The value of the paramter is an array of group names. For example: +# T_ONLY_PROCESS_THESE_GROUPS="['group-1','group-2']" T_ONLY_PROCESS_THESE_GROUPS="" -# Enter "true" if using a mysql container instead of an external database service such as AWS RDS. This will launch a mysql container. -T_USE_MYSQL_CONTAINER="" - # When CSV is generated, this determines how many form responses are held in memory during a batch. The higher the number the more memory this process will take but the faster it will complete. T_CSV_BATCH_SIZE=50 diff --git a/develop.sh b/develop.sh index a7af59e3cb..154a5c64c6 100755 --- a/develop.sh +++ b/develop.sh @@ -221,6 +221,7 @@ OPTIONS="--link $T_COUCHDB_CONTAINER_NAME:couchdb \ --volume $(pwd)/server/package.json:/tangerine/server/package.json:delegated \ --volume $(pwd)/server/src:/tangerine/server/src:delegated \ --volume $(pwd)/client/src:/tangerine/client/src:delegated \ + --volume $(pwd)/client/dist:/tangerine/client/dist:delegated \ --volume $(pwd)/server/reporting:/tangerine/server/reporting:delegated \ --volume $(pwd)/upgrades:/tangerine/upgrades:delegated \ --volume $(pwd)/scripts/generate-csv/bin.js:/tangerine/scripts/generate-csv/bin.js:delegated \ diff --git a/docs/developer/debugging-reporting.md b/docs/developer/debugging-reporting.md index a1f825ce1d..5d8c97db83 100644 --- a/docs/developer/debugging-reporting.md +++ b/docs/developer/debugging-reporting.md @@ -11,7 +11,7 @@ Summary of steps: 1. Enter the container on command line with `docker exec -it tangerine bash`. 1. Clear reporting cache with command `reporting-cache-clear`. 1. Run a batch with debugger enabled by running command `node --inspect-brk=0.0.0.0:9228 $(which reporting-worker-batch)`. -1. Latch onto debugging session using Chrome inspect. You may need to click "configure" and add `localhost:9228` to "Target discovery settings". +1. Latch onto debugging session using [Chrome Inspect](chrome://inspect/#devices). You may need to click "configure" and add `localhost:9228` to "Target discovery settings". ## Instructions diff --git a/docs/system-administrator/upgrade-instructions.md b/docs/system-administrator/upgrade-instructions.md new file mode 100644 index 0000000000..b0838f13b9 --- /dev/null +++ b/docs/system-administrator/upgrade-instructions.md @@ -0,0 +1,63 @@ +## Server upgrade instructions + +Reminder: Consider using the [Tangerine Upgrade Checklist](https://docs.tangerinecentral.org/system-administrator/upgrade-checklist.html) for making sure you test the upgrade safely. + +__Preparation__ + +Tangerine v3 images are relatively large, around 12GB. The server should have at least 20GB of free space plus the size of the data folder. Check the disk space before upgrading the the new version using the following steps: + +```bash +cd tangerine +# Check the size of the data folder. +du -sh data +# Check disk for free space. +df -h +``` + +If there is **less than** 20 GB plus the size of the data folder, create more space before proceeding. Good candidates to remove are: older versions of the Tangerine image and data backups. +```bash +# List all docker images. +docker image ls +# Remove the image of the version that is not being used. +docker rmi tangerine/tangerine: +# List all data backups. +ls -l data-backup-* +# Remove the data backups that are old and unneeded. +rm -rf ../data-backup- +``` + +__Upgrade__ + +After ensuring there is enough disk space, follow the steps below to upgrade the server. + +1. Backup the data folder +```bash +# Create a backup of the data folder. +cp -r data ../data-backup-$(date "+%F-%T") +``` + +2. Confirm there is no active synching from client devices + +Check logs for the past hour on the server to ensure it's not being actively used. Look for log messages like "Created sync session" for Devices that are syncing and "login success" for users logging in on the server. + +```bash +docker logs --since=60m tangerine +``` + +3. Install the new version of Tangerine +```bash +# Fetch the updates. +git fetch origin +# Checkout a new branch with the new version tag. +git checkout -b +# Run the start script with the new version. +./start.sh +``` + +__Clean Up__ + +After the upgrade, remove the previous version of the Tangerine image to free up disk space. + +```bash +docker rmi tangerine/tangerine: +``` \ No newline at end of file diff --git a/editor/package.json b/editor/package.json index 5437bf6419..f45a2e1fab 100644 --- a/editor/package.json +++ b/editor/package.json @@ -49,8 +49,8 @@ "redux": "^4.0.5", "rxjs": "~6.5.5", "rxjs-compat": "^6.5.5", - "tangy-form": "4.42.0", - "tangy-form-editor": "7.17.6", + "tangy-form": "^4.45.1", + "tangy-form-editor": "7.18.0", "translation-web-component": "0.0.3", "tslib": "^1.11.2", "ua-parser-js": "^0.7.24", diff --git a/editor/src/app/core/auth/_services/authentication.service.ts b/editor/src/app/core/auth/_services/authentication.service.ts index 2469d97612..1431f2b0b8 100644 --- a/editor/src/app/core/auth/_services/authentication.service.ts +++ b/editor/src/app/core/auth/_services/authentication.service.ts @@ -213,6 +213,10 @@ export class AuthenticationService { } async getCustomLoginMarkup() { - return await this.http.get('/custom-login-markup', {responseType: 'text'}).toPromise() + try { + return await this.http.get('/custom-login-markup', {responseType: 'text'}).toPromise() + } catch (error) { + return '' + } } } diff --git a/editor/src/app/groups/group-uploads-edit/group-uploads-edit.component.css b/editor/src/app/groups/group-uploads-edit/group-uploads-edit.component.css new file mode 100644 index 0000000000..e69de29bb2 diff --git a/editor/src/app/groups/group-uploads-edit/group-uploads-edit.component.html b/editor/src/app/groups/group-uploads-edit/group-uploads-edit.component.html new file mode 100644 index 0000000000..b88daa91a5 --- /dev/null +++ b/editor/src/app/groups/group-uploads-edit/group-uploads-edit.component.html @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/editor/src/app/groups/group-uploads-edit/group-uploads-edit.component.spec.ts b/editor/src/app/groups/group-uploads-edit/group-uploads-edit.component.spec.ts new file mode 100644 index 0000000000..e905d93244 --- /dev/null +++ b/editor/src/app/groups/group-uploads-edit/group-uploads-edit.component.spec.ts @@ -0,0 +1,25 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { GroupUploadsEditComponent } from './group-uploads-edit.component'; + +describe('GroupUploadsEditComponent', () => { + let component: GroupUploadsEditComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [ GroupUploadsEditComponent ] + }) + .compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(GroupUploadsEditComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/editor/src/app/groups/group-uploads-edit/group-uploads-edit.component.ts b/editor/src/app/groups/group-uploads-edit/group-uploads-edit.component.ts new file mode 100644 index 0000000000..953098eec1 --- /dev/null +++ b/editor/src/app/groups/group-uploads-edit/group-uploads-edit.component.ts @@ -0,0 +1,48 @@ +import { Component, OnInit, ViewChild } from '@angular/core'; +import { ActivatedRoute, Router } from '@angular/router'; +import { Breadcrumb } from 'src/app/shared/_components/breadcrumb/breadcrumb.component'; +import { _TRANSLATE } from 'src/app/shared/translation-marker'; +import { TangyFormsPlayerComponent } from 'src/app/tangy-forms/tangy-forms-player/tangy-forms-player.component'; + +@Component({ + selector: 'app-group-uploads-edit', + templateUrl: './group-uploads-edit.component.html', + styleUrls: ['./group-uploads-edit.component.css'] +}) +export class GroupUploadsEditComponent implements OnInit { + + title = _TRANSLATE('Edit Upload') + breadcrumbs:Array = [] + @ViewChild('formPlayer', {static: true}) formPlayer: TangyFormsPlayerComponent + + constructor( + private route:ActivatedRoute, + private router:Router + ) { } + + ngOnInit() { + this.route.params.subscribe(params => { + this.breadcrumbs = [ + { + label: _TRANSLATE('Uploads'), + url: 'uploads' + }, + { + label: _TRANSLATE('View Upload'), + url: `uploads/${params.responseId}` + }, + { + label: this.title, + url: `uploads/${params.responseId}/edit` + } + ] + this.formPlayer.formResponseId = params.responseId + this.formPlayer.render() + this.formPlayer.$submit.subscribe(async () => { + this.formPlayer.saveResponse(this.formPlayer.formEl.store.getState()) + this.router.navigate([`../`], { relativeTo: this.route }) + }) + }) + } + +} diff --git a/editor/src/app/groups/group-uploads-view/group-uploads-view.component.css b/editor/src/app/groups/group-uploads-view/group-uploads-view.component.css index e69de29bb2..9c05abd7b5 100644 --- a/editor/src/app/groups/group-uploads-view/group-uploads-view.component.css +++ b/editor/src/app/groups/group-uploads-view/group-uploads-view.component.css @@ -0,0 +1,12 @@ +.sticky { + position: sticky; + top: 20px; + max-width: fit-content; + margin-inline: auto; + z-index: 999; +} + +button, a{ + margin-right: 10px; + margin-top: 10px; +} \ No newline at end of file diff --git a/editor/src/app/groups/group-uploads-view/group-uploads-view.component.html b/editor/src/app/groups/group-uploads-view/group-uploads-view.component.html index ba6d57d67a..4d961bd6b5 100644 --- a/editor/src/app/groups/group-uploads-view/group-uploads-view.component.html +++ b/editor/src/app/groups/group-uploads-view/group-uploads-view.component.html @@ -1,2 +1,11 @@ - \ No newline at end of file +
+ + + + {{'Edit'|translate}} + + + +
+ diff --git a/editor/src/app/groups/group-uploads-view/group-uploads-view.component.ts b/editor/src/app/groups/group-uploads-view/group-uploads-view.component.ts index e74bace952..d0ef8fe862 100644 --- a/editor/src/app/groups/group-uploads-view/group-uploads-view.component.ts +++ b/editor/src/app/groups/group-uploads-view/group-uploads-view.component.ts @@ -3,6 +3,8 @@ import { ActivatedRoute, Router } from '@angular/router'; import { Breadcrumb } from './../../shared/_components/breadcrumb/breadcrumb.component'; import { _TRANSLATE } from 'src/app/shared/_services/translation-marker'; import { Component, OnInit, ViewChild } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { TangyFormService } from 'src/app/tangy-forms/tangy-form.service'; @Component({ selector: 'app-group-uploads-view', @@ -14,14 +16,21 @@ export class GroupUploadsViewComponent implements OnInit { title = _TRANSLATE('View Upload') breadcrumbs:Array = [] @ViewChild('formPlayer', {static: true}) formPlayer: TangyFormsPlayerComponent + responseId + groupId + formResponse constructor( private route:ActivatedRoute, - private router:Router + private router:Router, + private http: HttpClient, + private tangyFormService: TangyFormService, ) { } - ngOnInit() { - this.route.params.subscribe(params => { + async ngOnInit() { + this.route.params.subscribe(async params => { + this.responseId = params.responseId + this.groupId = params.groupId this.breadcrumbs = [ { label: _TRANSLATE('Uploads'), @@ -29,15 +38,83 @@ export class GroupUploadsViewComponent implements OnInit { }, { label: this.title, - url: `uploads/view/${params.responseId}` + url: `uploads/${params.responseId}` } ] this.formPlayer.formResponseId = params.responseId + this.formResponse = await this.tangyFormService.getResponse(params.responseId) this.formPlayer.render() this.formPlayer.$submit.subscribe(async () => { + this.formPlayer.saveResponse(this.formPlayer.formEl.store.getState()) this.router.navigate([`../`], { relativeTo: this.route }) }) }) } + async archive(){ + if(confirm('Are you sure you want to archive this form response?')) { + try { + const data = {...this.formPlayer.formEl.store.getState(), archived:true}; + const result = await this.tangyFormService.saveResponse(data) + if(result){ + alert(_TRANSLATE('Archived successfully.')) + this.router.navigate([`../`], { relativeTo: this.route }) + }else{ + alert(_TRANSLATE('Archival was unsuccessful. Please try again.')) + } + } catch (error) { + alert(_TRANSLATE('Archival was unsuccessful. Please try again.')) + console.log(error) + } + } + } + async unarchive(){ + if(confirm(_TRANSLATE('Are you sure you want to unarchive this form response?'))) { + try { + const data = {...this.formPlayer.formEl.store.getState(), archived:false}; + const result = await this.tangyFormService.saveResponse(data) + if(result){ + alert(_TRANSLATE('Unarchived successfully.')) + this.router.navigate([`../`], { relativeTo: this.route }) + }else{ + alert(_TRANSLATE('Unarchival was unsuccessful. Please try again.')) + } + } catch (error) { + alert(_TRANSLATE('Unarchival was unsuccessful. Please try again.')) + console.log(error) + } + } + } + + async verify(){ + try { + const data = {...this.formPlayer.formEl.store.getState(), verified:true}; + const result = await this.tangyFormService.saveResponse(data) + if(result){ + alert(_TRANSLATE('Verified successfully.')) + this.router.navigate([`../`], { relativeTo: this.route }) + }else{ + alert(_TRANSLATE('Verification was unsuccessful. Please try again.')) + } + } catch (error) { + alert(_TRANSLATE('Verification was unsuccessful. Please try again.')) + console.log(error) + } + } + async unverify(){ + try { + const data = {...this.formPlayer.formEl.store.getState(), verified:false}; + const result = await this.tangyFormService.saveResponse(data) + if(result){ + alert(_TRANSLATE('Unverified successfully.')) + this.router.navigate([`../`], { relativeTo: this.route }) + }else{ + alert(_TRANSLATE('Unverification was unsuccessful. Please try again.')) + } + } catch (error) { + alert(_TRANSLATE('Unverification was unsuccessful. Please try again.')) + console.log(error) + } + } + } diff --git a/editor/src/app/groups/group-uploads/group-uploads.component.html b/editor/src/app/groups/group-uploads/group-uploads.component.html index 7888162d83..4e6cd45b92 100644 --- a/editor/src/app/groups/group-uploads/group-uploads.component.html +++ b/editor/src/app/groups/group-uploads/group-uploads.component.html @@ -1,4 +1,4 @@
- +
diff --git a/editor/src/app/groups/groups-routing.module.ts b/editor/src/app/groups/groups-routing.module.ts index c63abca919..222e20421d 100644 --- a/editor/src/app/groups/groups-routing.module.ts +++ b/editor/src/app/groups/groups-routing.module.ts @@ -70,6 +70,7 @@ import { GroupCsvTemplatesComponent } from './group-csv-templates/group-csv-temp import { GroupDatabaseConflictsComponent } from './group-database-conflicts/group-database-conflicts.component'; import { DownloadStatisticalFileComponent } from './download-statistical-file/download-statistical-file.component'; import { GroupDevicePasswordPolicyComponent } from './group-device-password-policy/group-device-password-policy.component'; +import { GroupUploadsEditComponent } from './group-uploads-edit/group-uploads-edit.component'; const groupsRoutes: Routes = [ // { path: 'projects', component: GroupsComponent }, @@ -86,6 +87,7 @@ const groupsRoutes: Routes = [ { path: 'groups/:groupId/data', component: GroupDataComponent, canActivate: [LoginGuard] }, { path: 'groups/:groupId/data/uploads', component: GroupUploadsComponent, canActivate: [LoginGuard] }, { path: 'groups/:groupId/data/uploads/:responseId', component: GroupUploadsViewComponent, canActivate: [LoginGuard] }, + { path: 'groups/:groupId/data/uploads/:responseId/edit', component: GroupUploadsEditComponent, canActivate: [LoginGuard] }, { path: 'groups/:groupId/data/download-csv', component: GroupFormsCsvComponent, canActivate: [LoginGuard] }, { path: 'groups/:groupId/data/database-conflicts', component: GroupDatabaseConflictsComponent, canActivate: [LoginGuard] }, { path: 'groups/:groupId/data/csv-templates', component: GroupCsvTemplatesComponent, canActivate: [LoginGuard] }, diff --git a/editor/src/app/groups/groups.module.ts b/editor/src/app/groups/groups.module.ts index 9d6b54c97a..24511c1246 100644 --- a/editor/src/app/groups/groups.module.ts +++ b/editor/src/app/groups/groups.module.ts @@ -99,6 +99,7 @@ import { DownloadStatisticalFileComponent } from './download-statistical-file/do import { GroupDevicePasswordPolicyComponent } from './group-device-password-policy/group-device-password-policy.component'; import { GroupLocationListsComponent } from './group-location-lists/group-location-lists.component'; import { GroupLocationListNewComponent } from './group-location-list-new/group-location-list-new.component'; +import { GroupUploadsEditComponent } from './group-uploads-edit/group-uploads-edit.component'; @NgModule({ @@ -204,7 +205,8 @@ import { GroupLocationListNewComponent } from './group-location-list-new/group-l GroupCsvTemplatesComponent, GroupDatabaseConflictsComponent, DownloadStatisticalFileComponent, - GroupDevicePasswordPolicyComponent + GroupDevicePasswordPolicyComponent, + GroupUploadsEditComponent ], providers: [GroupsService, FilesService, TangerineFormsService, GroupDevicesService, TangyFormService ], }) diff --git a/editor/src/app/groups/responses/responses.component.html b/editor/src/app/groups/responses/responses.component.html index 55ba745427..dc2f796b0a 100644 --- a/editor/src/app/groups/responses/responses.component.html +++ b/editor/src/app/groups/responses/responses.component.html @@ -4,8 +4,13 @@ +
+ +
+
+
- +
< back -next > \ No newline at end of file +next > diff --git a/editor/src/app/groups/responses/responses.component.ts b/editor/src/app/groups/responses/responses.component.ts index eb03a49f82..3529df8b17 100644 --- a/editor/src/app/groups/responses/responses.component.ts +++ b/editor/src/app/groups/responses/responses.component.ts @@ -2,10 +2,14 @@ import { Router, ActivatedRoute } from '@angular/router'; import { AppConfigService } from 'src/app/shared/_services/app-config.service'; import { generateFlatResponse } from './tangy-form-response-flatten'; import { TangerineFormsService } from './../services/tangerine-forms.service'; -import { Component, OnInit, Input } from '@angular/core'; +import { Component, OnInit, Input,ViewChild, ElementRef } from '@angular/core'; import { GroupsService } from '../services/groups.service'; import { HttpClient } from '@angular/common/http'; import * as moment from 'moment' +import { t } from 'tangy-form/util/t.js' +import { Subject } from 'rxjs'; +import { debounceTime } from 'rxjs/operators'; +import { _TRANSLATE } from 'src/app/shared/translation-marker'; @Component({ selector: 'app-responses', @@ -19,6 +23,11 @@ export class ResponsesComponent implements OnInit { @Input() excludeForms:Array = [] @Input() excludeColumns:Array = [] @Input() hideFilterBy = false + @Input() hideActionBar = false + @Input() showArchiveButton = false + @ViewChild('searchBar', {static: true}) searchBar: ElementRef + @ViewChild('searchResults', {static: true}) searchResults: ElementRef + onSearch$ = new Subject() ready = false @@ -27,6 +36,9 @@ export class ResponsesComponent implements OnInit { skip = 0; limit = 30; forms = []; + locationLists + searchString + initialResults = [] constructor( private groupsService: GroupsService, @@ -43,12 +55,42 @@ export class ResponsesComponent implements OnInit { this.forms = (await this.tangerineFormsService.getFormsInfo(this.groupId)) .filter(formInfo => !this.excludeForms.includes(formInfo.id) ) await this.getResponses() - this.ready = true + this.ready = true; + this.onSearch$ + .pipe(debounceTime(300)) + .subscribe((searchString:string) => { + this.responses.length <= 0 ? this.searchResults.nativeElement.innerHTML = 'Searching...' : null + this.onSearch(searchString) + }); + this + .searchBar + .nativeElement + .addEventListener('keyup', async event => { + const searchString = event.target.value.trim() + if (searchString.length > 2) { + if (this.searchString === searchString) return; + this.searchResults.nativeElement.innerHTML = 'Searching...' + this.skip = 0 + this.limit = 30 + this.onSearch$.next(event.target.value) + } if(searchString.length <= 2 && searchString.length !==0 ) { + this.searchResults.nativeElement.innerHTML = ` + + ${t('Enter more than two characters...')} + + ` + } if(searchString.length===0){ + this.searchResults.nativeElement.innerHTML = '' + this.searchString = '' + this.skip = 0 + this.limit = 30 + await this.getResponses() + } + }) } async getResponses() { - const data: any = await this.groupsService.getLocationLists(this.groupId); - const locationLists = data; + this.locationLists = await this.groupsService.getLocationLists(this.groupId); let responses = [] if (this.filterBy === '*') { responses = >await this.http.get(`/api/${this.groupId}/responses/${this.limit}/${this.skip}`).toPromise() @@ -57,11 +99,31 @@ export class ResponsesComponent implements OnInit { } const flatResponses = [] for (let response of responses) { - const flatResponse = await generateFlatResponse(response, locationLists, false) + const flatResponse = await generateFlatResponse(response, this.locationLists, false) this.excludeColumns.forEach(column => delete flatResponse[column]) flatResponses.push(flatResponse) } this.responses = flatResponses + this.initialResults = flatResponses + } + + async onSearch(searchString) { + this.searchString = searchString + this.responses = [] + if (searchString === '') { + this.searchResults.nativeElement.innerHTML = '' + this.responses = this.initialResults + return + } + const responses = >await this.http.get(`/api/${this.groupId}/responses/${this.limit}/${this.skip}/?id=${searchString}`).toPromise() + const flatResponses = [] + for (let response of responses) { + const flatResponse = await generateFlatResponse(response, this.locationLists, false) + this.excludeColumns.forEach(column => delete flatResponse[column]) + flatResponses.push(flatResponse) + } + this.responses = flatResponses + this.responses.length > 0 ? this.searchResults.nativeElement.innerHTML = '': this.searchResults.nativeElement.innerHTML = 'No results matching the criteria.' } async filterByForm(event) { @@ -69,30 +131,47 @@ export class ResponsesComponent implements OnInit { this.skip = 0; this.getResponses(); } - - async deleteResponse(id) { - if(confirm('Are you sure you want to delete this form response?')) { - await this.http.delete(`/api/${this.groupId}/${id}`).toPromise() - this.getResponses() + async archiveResponse(id) { + try { + if(confirm(_TRANSLATE('Are you sure you want to archive this form response?'))) { + const result = await this.http.patch(`/group-responses/patch/${this.groupId}/${id}`,{archived:true}).toPromise() + if(result){ + alert(_TRANSLATE('Archived successfully.')) + this.getResponses() + }else{ + alert(_TRANSLATE('Archival was unsuccessful. Please try again.')) + } } + } catch (error) { + alert(_TRANSLATE('Archival was unsuccessful. Please try again.')) + console.log(error) + } } - nextPage() { this.skip = this.skip + this.limit - this.getResponses(); + if(this.searchString){ + this.onSearch(this.searchString) + } else{ + this.getResponses(); + } } previousPage() { this.skip = this.skip - this.limit + if(this.searchString){ + this.onSearch(this.searchString) + } else{ this.getResponses(); + } } onRowEdit(row) { this.router.navigate([row._id ? row._id : row.id], {relativeTo: this.route}) } - - onRowDelete(row) { - this.deleteResponse(row._id ? row._id : row.id) + onRowArchive(row) { + this.archiveResponse(row._id ? row._id : row.id) + } + onRowClick(row){ + this.router.navigate([row._id ? row._id : row.id], {relativeTo: this.route}) } - } diff --git a/editor/src/app/groups/responses/tangy-form-response-flatten.ts b/editor/src/app/groups/responses/tangy-form-response-flatten.ts index 6b5dc723aa..411dd9314e 100644 --- a/editor/src/app/groups/responses/tangy-form-response-flatten.ts +++ b/editor/src/app/groups/responses/tangy-form-response-flatten.ts @@ -22,6 +22,9 @@ export const generateFlatResponse = async function (formResponse, locationLists, deviceId: formResponse.deviceId||'', groupId: formResponse.groupId||'', complete: formResponse.complete, + verified: formResponse.verified||'', + archived: formResponse.archived||'', + tangerineModifiedOn: formResponse.tangerineModifiedOn||'', tangerineModifiedByUserId: formResponse.tangerineModifiedByUserId, ...formResponse.caseId ? { caseId: formResponse.caseId, diff --git a/editor/src/app/ng-tangy-form-editor/feedback-editor/feedback-editor.component.html b/editor/src/app/ng-tangy-form-editor/feedback-editor/feedback-editor.component.html index ba4c160eae..bfa9f1dae1 100644 --- a/editor/src/app/ng-tangy-form-editor/feedback-editor/feedback-editor.component.html +++ b/editor/src/app/ng-tangy-form-editor/feedback-editor/feedback-editor.component.html @@ -89,6 +89,9 @@

Tasks/Subtasks with Feedback

+ + +
diff --git a/editor/src/app/ng-tangy-form-editor/feedback-editor/feedback-editor.component.ts b/editor/src/app/ng-tangy-form-editor/feedback-editor/feedback-editor.component.ts index 90930f35c9..78fd4c939b 100644 --- a/editor/src/app/ng-tangy-form-editor/feedback-editor/feedback-editor.component.ts +++ b/editor/src/app/ng-tangy-form-editor/feedback-editor/feedback-editor.component.ts @@ -25,6 +25,7 @@ export class FeedbackEditorComponent implements OnInit { skill:string = "" assignment:string = "" message:string = "" + customJSCode:string = "" formItems:any formItem:string formItemName:string @@ -122,6 +123,7 @@ export class FeedbackEditorComponent implements OnInit { this.skill = "" this.assignment = "" this.message = "" + this.customJSCode = "" this.formItem = "" this.showFeedbackForm = true @@ -140,6 +142,7 @@ export class FeedbackEditorComponent implements OnInit { this.feedback.skill = this.skill this.feedback.assignment = this.assignment this.feedback.message = this.message + this.feedback.customJSCode = this.customJSCode this.feedback.formItem = this.formItem this.feedbackService.update(this.groupName, this.formId, this.feedback) .then(async (data) => { @@ -168,6 +171,7 @@ export class FeedbackEditorComponent implements OnInit { this.skill = this.feedback.skill this.assignment = this.feedback.assignment this.message = this.feedback.message + this.customJSCode = this.feedback.customJSCode this.formItem = this.feedback.formItem this.percentileOptions = this.createPercentileOptions(null) diff --git a/editor/src/app/ng-tangy-form-editor/feedback-editor/feedback.ts b/editor/src/app/ng-tangy-form-editor/feedback-editor/feedback.ts index 3523989abe..97adf98a6b 100644 --- a/editor/src/app/ng-tangy-form-editor/feedback-editor/feedback.ts +++ b/editor/src/app/ng-tangy-form-editor/feedback-editor/feedback.ts @@ -8,5 +8,6 @@ export class Feedback { skill:string; assignment:string; message:string; + customJSCode: string; messageTruncated: string; // for listing } diff --git a/editor/src/app/shared/_components/dynamic-table/dynamic-table.component.css b/editor/src/app/shared/_components/dynamic-table/dynamic-table.component.css index 0081f73aa7..22016e53ea 100644 --- a/editor/src/app/shared/_components/dynamic-table/dynamic-table.component.css +++ b/editor/src/app/shared/_components/dynamic-table/dynamic-table.component.css @@ -36,3 +36,6 @@ mat-cell, mat-header-cell { .mat-header-cell, .mat-cell { padding-left: 5px; } +.mat-row{ + cursor: pointer; +} \ No newline at end of file diff --git a/editor/src/app/shared/_components/dynamic-table/dynamic-table.component.html b/editor/src/app/shared/_components/dynamic-table/dynamic-table.component.html index 2ceba62697..0b632daf5c 100644 --- a/editor/src/app/shared/_components/dynamic-table/dynamic-table.component.html +++ b/editor/src/app/shared/_components/dynamic-table/dynamic-table.component.html @@ -5,19 +5,22 @@ {{ column.cell(row) }} - + -
+
- +
@@ -26,6 +29,6 @@ - + - \ No newline at end of file + diff --git a/editor/src/app/shared/_components/dynamic-table/dynamic-table.component.ts b/editor/src/app/shared/_components/dynamic-table/dynamic-table.component.ts index 9be8200668..63d8f68c67 100644 --- a/editor/src/app/shared/_components/dynamic-table/dynamic-table.component.ts +++ b/editor/src/app/shared/_components/dynamic-table/dynamic-table.component.ts @@ -11,8 +11,11 @@ export class DynamicTableComponent implements OnInit, OnChanges { @Output() rowClick: EventEmitter = new EventEmitter(); @Output() rowEdit: EventEmitter = new EventEmitter(); @Output() rowDelete: EventEmitter = new EventEmitter(); + @Output() rowArchive: EventEmitter = new EventEmitter(); @Input() data:Array = [] @Input() columnLabels = {} + @Input() hideActionBar + @Input() showArchiveButton columns:Array displayedColumns:Array dataSource:any @@ -44,22 +47,28 @@ export class DynamicTableComponent implements OnInit, OnChanges { cell: (element: any) => `${element[column] ? element[column] : ``}` } }) - this.displayedColumns = [...this.columns.map(c => c.columnDef), 'actions']; + this.displayedColumns = this.hideActionBar? [...this.columns.map(c => c.columnDef)]:[...this.columns.map(c => c.columnDef), 'actions'] // Set the dataSource for . this.dataSource = this.data } - onRowClick(row) { - this.rowClick.emit(row) - } + onRowClick(event,row) { + if (event.target.tagName === 'MAT-ICON'){ + event.stopPropagation() + }else{ + this.rowClick.emit(row) + } + } onRowEdit(row) { this.rowEdit.emit(row) } - onRowDelete(row) { this.rowDelete.emit(row) } + onRowArchive(row) { + this.rowArchive.emit(row) + } diff --git a/editor/src/app/shared/shared.module.ts b/editor/src/app/shared/shared.module.ts index dbec5cc0c8..14307583f4 100644 --- a/editor/src/app/shared/shared.module.ts +++ b/editor/src/app/shared/shared.module.ts @@ -25,15 +25,16 @@ import {MatProgressBarModule} from "@angular/material/progress-bar"; @NgModule({ schemas: [CUSTOM_ELEMENTS_SCHEMA], - imports: [ - CommonModule, - MatTableModule, - MatMenuModule, - MatIconModule, - MatDialogModule, - MatButtonModule, - MatProgressBarModule - ], + imports: [ + CommonModule, + MatTableModule, + MatMenuModule, + MatIconModule, + MatDialogModule, + MatButtonModule, + MatProgressBarModule, + TranslateModule + ], providers: [ AppConfigService, ServerConfigService, @@ -42,7 +43,6 @@ import {MatProgressBarModule} from "@angular/material/progress-bar"; ProcessGuard ], exports: [ - TranslateModule, DynamicTableComponent, MatSnackBarModule, TangyLoadingComponent, @@ -51,7 +51,8 @@ import {MatProgressBarModule} from "@angular/material/progress-bar"; NgxPermissionsModule, HasAPermissionDirective, HasSomePermissionsDirective, - HasAllPermissionsDirective + HasAllPermissionsDirective, + TranslateModule ], declarations: [ TangyLoadingComponent, diff --git a/editor/src/app/tangy-forms/tangy-forms-player/tangy-forms-player.component.ts b/editor/src/app/tangy-forms/tangy-forms-player/tangy-forms-player.component.ts index 492bee5636..ff52dc5131 100755 --- a/editor/src/app/tangy-forms/tangy-forms-player/tangy-forms-player.component.ts +++ b/editor/src/app/tangy-forms/tangy-forms-player/tangy-forms-player.component.ts @@ -25,7 +25,8 @@ export class TangyFormsPlayerComponent { // 2. Use this if you want to attach the form response yourself. @Input('response') response:TangyFormResponseModel // 3. Use this is you want a new form response. - @Input('formId') formId:string + @Input('formId') formId:string + @Input('unlockFormResponses') unlockFormResponses:boolean @Input('templateId') templateId:string @Input('location') location:any @@ -193,6 +194,7 @@ export class TangyFormsPlayerComponent { this.$afterResubmit.next(true) }) } + this.unlockFormResponses? this.formEl.unlock({disableComponents:['TANGY-GPS']}): null this.$rendered.next(true) this.rendered = true } diff --git a/online-survey-app/package.json b/online-survey-app/package.json index e3146f09a8..a958912eb8 100644 --- a/online-survey-app/package.json +++ b/online-survey-app/package.json @@ -25,7 +25,7 @@ "material-design-icons-iconfont": "^6.1.0", "redux": "^4.0.5", "rxjs": "~6.6.0", - "tangy-form": "v4.42.0", + "tangy-form": "^4.43.2", "translation-web-component": "^1.1.0", "tslib": "^2.0.0", "zone.js": "~0.10.2" diff --git a/online-survey-app/src/polyfills.ts b/online-survey-app/src/polyfills.ts index 694f825381..9be4cf8b9b 100644 --- a/online-survey-app/src/polyfills.ts +++ b/online-survey-app/src/polyfills.ts @@ -89,6 +89,7 @@ import 'tangy-form/input/tangy-signature.js'; import 'tangy-form/input/tangy-toggle.js'; import 'tangy-form/input/tangy-radio-blocks.js'; import 'tangy-form/input/tangy-video-capture.js'; +import 'tangy-form/input/tangy-prompt-box.js'; import 'translation-web-component/t-select.js' diff --git a/server/package.json b/server/package.json index ca98b895bb..881db6d507 100644 --- a/server/package.json +++ b/server/package.json @@ -29,6 +29,7 @@ "update-group-archived-index": "./src/scripts/update-group-archived-index.js", "find-missing-records": "./src/scripts/find-missing-records.js", "import-archives": "./src/scripts/import-archives/bin.js", + "import-archives-v2": "./src/scripts/import-archives-v2/bin.js", "reset-all-devices": "./src/scripts/reset-all-devices/bin.js", "translations-update": "./src/scripts/translations-update.js", "release-dat": "./src/scripts/release-dat.sh", diff --git a/server/src/core/group-responses/group-responses.controller.ts b/server/src/core/group-responses/group-responses.controller.ts index 0a88ec64f1..1e7d683bb4 100644 --- a/server/src/core/group-responses/group-responses.controller.ts +++ b/server/src/core/group-responses/group-responses.controller.ts @@ -1,6 +1,8 @@ import { GroupResponsesService } from './../../shared/services/group-responses/group-responses.service'; -import { Controller, All, Param, Body } from '@nestjs/common'; +import { Controller, All, Param, Body , Req} from '@nestjs/common'; import { SSL_OP_TLS_BLOCK_PADDING_BUG } from 'constants'; +import { Request } from 'express'; +import { decodeJWT } from 'src/auth-utils'; const log = require('tangy-log').log @Controller('group-responses') @@ -19,7 +21,6 @@ export class GroupResponsesController { async query(@Param('groupId') groupId, @Body('query') query) { return await this.groupResponsesService.find(groupId, query) } - @All('search/:groupId') async search(@Param('groupId') groupId, @Body('phrase') phrase, @Body('index') index) { return await this.groupResponsesService.search(groupId, phrase, index) @@ -48,8 +49,9 @@ export class GroupResponsesController { } @All('update/:groupId') - async update(@Param('groupId') groupId, @Body('response') response:any) { - const freshResponse = await this.groupResponsesService.update(groupId, response) + async update(@Param('groupId') groupId, @Body('response') response:any, @Req() request:Request) { + const tangerineModifiedByUserId = decodeJWT(request['headers']['authorization'])['username'] + const freshResponse = await this.groupResponsesService.update(groupId, {...response, tangerineModifiedByUserId}) return freshResponse } @@ -58,5 +60,11 @@ export class GroupResponsesController { await this.groupResponsesService.delete(groupId, responseId) return {} } - + @All('patch/:groupId/:responseId') + async patch(@Param('groupId') groupId:string, @Param('responseId') responseId:string, @Req() request:Request) { + const tangerineModifiedByUserId = decodeJWT(request['headers']['authorization'])['username'] + const doc = await this.groupResponsesService.read(groupId, responseId) + const freshResponse = await this.groupResponsesService.update(groupId, {...doc,...request['body'],tangerineModifiedByUserId}) + return request['body'] + } } diff --git a/server/src/express-app.js b/server/src/express-app.js index ad69f9cd50..56bd30a618 100644 --- a/server/src/express-app.js +++ b/server/src/express-app.js @@ -210,6 +210,7 @@ app.get('/app/:groupId/responsesByMonthAndFormId/:keys/:limit?/:skip?', isAuthen // Note that the lack of security middleware here is intentional. User IDs are UUIDs and thus sufficiently hard to guess. app.get('/api/:groupId/responsesByUserProfileId/:userProfileId/:limit?/:skip?', require('./routes/group-responses-by-user-profile-id.js')) app.get('/api/:groupId/responsesByUserProfileShortCode/:userProfileShortCode/:limit?/:skip?', require('./routes/group-responses-by-user-profile-short-code.js')) +app.get('/api/:groupId/userProfileByShortCode/:userProfileShortCode', require('./routes/group-user-profile-by-short-code.js')) app.get('/api/:groupId/:docId', isAuthenticatedOrHasUploadToken, require('./routes/group-doc-read.js')) app.put('/api/:groupId/:docId', isAuthenticated, require('./routes/group-doc-write.js')) app.post('/api/:groupId/:docId', isAuthenticated, require('./routes/group-doc-write.js')) diff --git a/server/src/group-views.js b/server/src/group-views.js index 5d9690cb3c..8b979c3b9e 100644 --- a/server/src/group-views.js +++ b/server/src/group-views.js @@ -59,10 +59,10 @@ module.exports.unpaid = function(doc) { } } -module.exports.responsesByUserProfileShortCode = function(doc) { - if (doc.collection === "TangyFormResponse") { +module.exports.responsesByUserProfileShortCode = { + map: function (doc) { if (doc.form && doc.form.id === 'user-profile') { - return emit(doc._id.substr(doc._id.length-6, doc._id.length), true) + return emit(doc._id.substr(doc._id.length-6, doc._id.length), 1) } var inputs = doc.items.reduce(function(acc, item) { return acc.concat(item.inputs)}, []) var userProfileInput = null @@ -72,8 +72,15 @@ module.exports.responsesByUserProfileShortCode = function(doc) { } }) if (userProfileInput) { - emit(userProfileInput.value.substr(userProfileInput.value.length-6, userProfileInput.value.length), true) + emit(userProfileInput.value.substr(userProfileInput.value.length-6, userProfileInput.value.length), 1) } + }, + reduce: '_count' +} + +module.exports.userProfileByUserProfileShortCode = function (doc) { + if (doc.collection === "TangyFormResponse" && doc.form && doc.form.id === 'user-profile') { + return emit(doc._id.substr(doc._id.length - 6, doc._id.length), true); } } @@ -214,4 +221,4 @@ module.exports.byConflictDocId = { emit(doc.conflictDocId, doc.conflictRev); }.toString(), reduce: '_count' -} \ No newline at end of file +} diff --git a/server/src/modules/csv/index.js b/server/src/modules/csv/index.js index 0e47189a7a..e71ca69d1e 100644 --- a/server/src/modules/csv/index.js +++ b/server/src/modules/csv/index.js @@ -243,9 +243,11 @@ const generateFlatResponse = async function (formResponse, sanitized, groupId) deviceId: formResponse.deviceId || '', groupId: formResponse.groupId || '', complete: formResponse.complete, + verified: formResponse?.verified||'', + tangerineModifiedOn: formResponse?.tangerineModifiedOn||'', // NOTE: Doubtful that anything with an archived flag would show up here because it would have been deleted already in 'Delete from the -reporting db.' - archived: formResponse.archived, - tangerineModifiedByUserId: formResponse.tangerineModifiedByUserId, + archived: formResponse?.archived||'', + tangerineModifiedByUserId: formResponse?.tangerineModifiedByUserId||'', ...formResponse.caseId ? { caseId: formResponse.caseId, eventId: formResponse.eventId, @@ -260,6 +262,7 @@ const generateFlatResponse = async function (formResponse, sanitized, groupId) flatFormResponse['grade'] = formResponse.grade flatFormResponse['schoolName'] = formResponse.schoolName flatFormResponse['schoolYear'] = formResponse.schoolYear + flatFormResponse['reportDate'] = formResponse.reportDate flatFormResponse['type'] = formResponse.type if (formResponse.type === 'attendance') { flatFormResponse['attendanceList'] = formResponse.attendanceList diff --git a/server/src/modules/logstash/index.js b/server/src/modules/logstash/index.js index 3f6a3d7fc5..a516f9cc96 100644 --- a/server/src/modules/logstash/index.js +++ b/server/src/modules/logstash/index.js @@ -115,7 +115,11 @@ const generateFlatResponse = async function (formResponse, locationList) { buildChannel: formResponse.buildChannel||'', deviceId: formResponse.deviceId||'', groupId: formResponse.groupId||'', - complete: formResponse.complete + complete: formResponse.complete, + verified: formResponse?.verified||'', + tangerineModifiedOn: formResponse?.tangerineModifiedOn||'', + tangerineModifiedByUserId: formResponse.tangerineModifiedByUserId||'', + archived: formResponse?.archived||'', }; let formID = formResponse.form.id; diff --git a/server/src/modules/mysql-js/index.js b/server/src/modules/mysql-js/index.js index 06f7e5c027..54970dba3c 100644 --- a/server/src/modules/mysql-js/index.js +++ b/server/src/modules/mysql-js/index.js @@ -206,14 +206,31 @@ module.exports = { } const result = await saveToMysql(knex, sourceDb,flatDoc, tablenameSuffix, tableName, docType, primaryKey, createFunction) log.info('Processed: ' + JSON.stringify(result)) - } else { + } else if (doc.type === 'response') { const flatDoc = await prepareFlatData(doc, sanitized); tableName = null; docType = 'response'; primaryKey = 'ID' createFunction = function (t) { t.engine('InnoDB') - t.string(primaryKey, 36).notNullable().primary(); + t.string(primaryKey, 200).notNullable().primary(); + t.string('caseId', 36) // .index('response_caseId_IDX'); + t.string('participantID', 36) //.index('case_instances_ParticipantID_IDX'); + t.string('caseEventId', 36) // .index('eventform_caseEventId_IDX'); + t.tinyint('complete'); + t.string('archived', 36); // TODO: "sqlMessage":"Incorrect integer value: '' for column 'archived' at row 1 + } + const result = await saveToMysql(knex, sourceDb,flatDoc, tablenameSuffix, tableName, docType, primaryKey, createFunction) + log.info('Processed: ' + JSON.stringify(result)) + } else { + const flatDoc = await prepareFlatData(doc, sanitized); + tableName = flatDoc.type; + console.log("tableName: " + tableName) + docType = 'response'; + primaryKey = 'ID' + createFunction = function (t) { + t.engine('InnoDB') + t.string(primaryKey, 200).notNullable().primary(); t.string('caseId', 36) // .index('response_caseId_IDX'); t.string('participantID', 36) //.index('case_instances_ParticipantID_IDX'); t.string('caseEventId', 36) // .index('eventform_caseEventId_IDX'); @@ -223,6 +240,7 @@ module.exports = { const result = await saveToMysql(knex, sourceDb,flatDoc, tablenameSuffix, tableName, docType, primaryKey, createFunction) log.info('Processed: ' + JSON.stringify(result)) } + await knex.destroy() } } @@ -310,6 +328,7 @@ const getItemValue = (doc, variableName) => { const generateFlatResponse = async function (formResponse, sanitized) { const groupId = formResponse.groupId; + // Anything added to this list need to be added to the valuesToRemove list in each of the convert_response function. let flatFormResponse = { _id: formResponse._id, formTitle: formResponse.form.title, @@ -319,7 +338,10 @@ const generateFlatResponse = async function (formResponse, sanitized) { deviceId: formResponse.deviceId||'', groupId: groupId||'', complete: formResponse.complete, - archived: formResponse.archived||'' + archived: formResponse?.archived||'', + tangerineModifiedOn: formResponse?.tangerineModifiedOn||'', + tangerineModifiedByUserId: formResponse?.tangerineModifiedByUserId||'', + verified: formResponse?.verified||'' }; if (!formResponse.formId) { @@ -329,6 +351,52 @@ const generateFlatResponse = async function (formResponse, sanitized) { flatFormResponse['formId'] = formResponse.form.id } + if (formResponse.type === 'attendance' || formResponse.type === 'behavior' || formResponse.type === 'scores' || + formResponse.form.id === 'student-registration' || + formResponse.form.id === 'class-registration') { + + function hackFunctionToRemoveUserProfileId (formResponse) { + // This is a very special hack function to remove userProfileId + // It needs to be replaced with a proper solution that resolves duplicate variables. + if (formResponse.userProfileId) { + for (let item of formResponse.items) { + for (let input of item.inputs) { + if (input.name === 'userProfileId') { + delete formResponse.userProfileId; + } + } + } + } + return formResponse; + } + + formResponse = hackFunctionToRemoveUserProfileId(formResponse); + + if (formResponse.type === 'attendance') { + flatFormResponse['attendanceList'] = formResponse.attendanceList + } else if (formResponse.type === 'behavior') { + // flatFormResponse['studentBehaviorList'] = formResponse.studentBehaviorList + const studentBehaviorList = formResponse.studentBehaviorList.map(record => { + const student = {} + Object.keys(record).forEach(key => { + if (key === 'behavior') { + student[key + '.formResponseId'] = record[key]['formResponseId'] + student[key + '.internal'] = record[key]['internal'] + student[key + '.internalPercentage'] = record[key]['internalPercentage'] + // console.log("special processing for behavior: " + JSON.stringify(student) ) + } else { + // console.log("key: " + key + " record[key]: " + record[key]) + student[key] = record[key] + } + }) + return student + }) + flatFormResponse['studentBehaviorList'] = studentBehaviorList + } else if (formResponse.type === 'scores') { + flatFormResponse['scoreList'] = formResponse.scoreList + } + } + for (let item of formResponse.items) { for (let input of item.inputs) { let sanitize = false; @@ -713,7 +781,7 @@ async function convert_response(knex, doc, groupId, tableName) { // # Delete the following keys; - const valuesToRemove = ['_id', '_rev','buildChannel','buildId','caseEventId','deviceId','eventFormId','eventId','groupId','participantId','startDatetime', 'startUnixtime'] + const valuesToRemove = ['_id', '_rev','buildChannel','buildId','caseEventId','deviceId','eventFormId','eventId','groupId','participantId','startDatetime', 'startUnixtime', 'tangerineModifiedOn', 'tangerineModifiedByUserId'] valuesToRemove.forEach(e => delete doc[e]); const cleanData = populateDataFromDocument(doc, data); return cleanData @@ -817,6 +885,9 @@ async function saveToMysql(knex, sourceDb, doc, tablenameSuffix, tableName, docT data = await convert_issue(knex, doc, groupId, tableName) break; case 'response': + case 'attendance': + case 'behavior': + case 'scores': data = await convert_response(knex, doc, groupId, tableName) // Check if table exists and create if needed: tableName = data['formID_sanitized'] + tablenameSuffix diff --git a/server/src/modules/mysql/index.js b/server/src/modules/mysql/index.js index a630337c59..a7ef66fba5 100644 --- a/server/src/modules/mysql/index.js +++ b/server/src/modules/mysql/index.js @@ -266,7 +266,10 @@ const generateFlatResponse = async function (formResponse, sanitized) { deviceId: formResponse.deviceId||'', groupId: formResponse.groupId||'', complete: formResponse.complete, - archived: formResponse.archived||'' + archived: formResponse?.archived||'', + tangerineModifiedOn: formResponse?.tangerineModifiedOn||'', + tangerineModifiedByUserId: formResponse.tangerineModifiedByUserId|'', + verified: formResponse?.verified|'', }; for (let item of formResponse.items) { for (let input of item.inputs) { diff --git a/server/src/modules/rshiny/index.js b/server/src/modules/rshiny/index.js index c31ffa8722..53c8a3bfca 100644 --- a/server/src/modules/rshiny/index.js +++ b/server/src/modules/rshiny/index.js @@ -161,7 +161,11 @@ const generateFlatResponse = async function (formResponse, locationList, sanitiz buildChannel: formResponse.buildChannel||'', deviceId: formResponse.deviceId||'', groupId: formResponse.groupId||'', - complete: formResponse.complete + complete: formResponse.complete, + archived: formResponse?.archived||'', + tangerineModifiedOn: formResponse?.tangerineModifiedOn||'', + tangerineModifiedByUserId: formResponse?.tangerineModifiedByUserId||'', + verified: formResponse?.verified||'' }; function set(input, key, value) { flatFormResponse[key] = input.skipped diff --git a/server/src/modules/synapse/index.js b/server/src/modules/synapse/index.js index 7b48a5f3ba..c3bb54ec76 100644 --- a/server/src/modules/synapse/index.js +++ b/server/src/modules/synapse/index.js @@ -186,7 +186,11 @@ const generateFlatResponse = async function (formResponse, locationList, sanitiz buildChannel: formResponse.buildChannel||'', deviceId: formResponse.deviceId||'', groupId: formResponse.groupId||'', - complete: formResponse.complete + complete: formResponse.complete, + archived: formResponse?.archived||'', + tangerineModifiedOn: formResponse?.tangerineModifiedOn||'', + tangerineModifiedByUserId: formResponse?.tangerineModifiedByUserId||'', + verified: formResponse?.verified||'', }; function set(input, key, value) { flatFormResponse[key] = input.skipped diff --git a/server/src/reporting/clear-reporting-cache.js b/server/src/reporting/clear-reporting-cache.js index 071ef04868..0d529440bf 100644 --- a/server/src/reporting/clear-reporting-cache.js +++ b/server/src/reporting/clear-reporting-cache.js @@ -24,7 +24,16 @@ async function clearReportingCache() { if (reportingWorkerRunning) await sleep(1*1000) } console.log('Clearing reporting caches...') - const groupNames = await groupsList() + let groupNames = await groupsList() + + let onlyProcessTheseGroups = [] + if (process.env.T_ONLY_PROCESS_THESE_GROUPS && process.env.T_ONLY_PROCESS_THESE_GROUPS !== '') { + onlyProcessTheseGroups = process.env.T_ONLY_PROCESS_THESE_GROUPS + ? JSON.parse(process.env.T_ONLY_PROCESS_THESE_GROUPS.replace(/\'/g, `"`)) + : [] + } + groupNames = groupNames.filter(groupName => onlyProcessTheseGroups.includes(groupName)); + await tangyModules.hook('clearReportingCache', { groupNames }) // update worker state console.log('Resetting reporting worker state...') @@ -33,7 +42,6 @@ async function clearReportingCache() { const newState = Object.assign({}, state, { databases: state.databases.map(({name, sequence}) => { return {name, sequence: 0}}) }) - console.log("newState: " + JSON.stringify(newState)) await writeFile(REPORTING_WORKER_STATE, JSON.stringify(newState), 'utf-8') await unlink(REPORTING_WORKER_PAUSE) console.log('Done!') diff --git a/server/src/routes/group-responses-by-user-profile-short-code.js b/server/src/routes/group-responses-by-user-profile-short-code.js index 2bf65f469c..19b82593b7 100644 --- a/server/src/routes/group-responses-by-user-profile-short-code.js +++ b/server/src/routes/group-responses-by-user-profile-short-code.js @@ -1,20 +1,25 @@ const DB = require('../db.js') -const clog = require('tangy-log').clog const log = require('tangy-log').log module.exports = async (req, res) => { try { const groupDb = new DB(req.params.groupId) - let options = {key: req.params.userProfileShortCode, include_docs: true} + const userProfileShortCode = req.params.userProfileShortCode + let options = { key: userProfileShortCode } if (req.params.limit) { options.limit = req.params.limit } if (req.params.skip) { options.skip = req.params.skip } - const results = await groupDb.query('responsesByUserProfileShortCode', options); - const docs = results.rows.map(row => row.doc) - res.send(docs) + if (req.query.totalRows) { + const results = await groupDb.query('responsesByUserProfileShortCode', { key: userProfileShortCode, limit: 1,skip: 0, include_docs: false, reduce: true, group: true }); + res.send({ totalDocs: results.rows[0].value }) + } else { + const results = await groupDb.query('responsesByUserProfileShortCode', { ...options, include_docs: true, reduce: false }); + const docs = results.rows.map(row => row.doc) + res.send(docs) + } } catch (error) { log.error(error); res.status(500).send(error); diff --git a/server/src/routes/group-responses.js b/server/src/routes/group-responses.js index e8b00a810a..ff9155f941 100644 --- a/server/src/routes/group-responses.js +++ b/server/src/routes/group-responses.js @@ -5,16 +5,29 @@ const log = require('tangy-log').log module.exports = async (req, res) => { try { const groupDb = new DB(req.params.groupId) - let options = {key: req.params.formId, include_docs: true, descending: true} + let options = {include_docs: true} if (req.params.limit) { options.limit = req.params.limit } if (req.params.skip) { options.skip = req.params.skip } - const results = await groupDb.query('responsesByStartUnixTime', options); - const docs = results.rows.map(row => row.doc) - res.send(docs) + let results = undefined; + if (Object.keys(req.query).length < 1 ) { + // get all responses for a form + options.key = req.params.formId; + options.descending = true; + results = await groupDb.query('responsesByStartUnixTime', options); + } else { + // searh options by document id + options.startkey = req.query.id; + options.endkey = `${req.query.id}\ufff0`; + results = await groupDb.allDocs(options); + } + if (results) { + const docs = results.rows.map(row => { if (row.doc && row.doc.collection == "TangyFormResponse") { return row.doc } }); + res.send(docs) + } } catch (error) { log.error(error); res.status(500).send(error); diff --git a/server/src/routes/group-user-profile-by-short-code.js b/server/src/routes/group-user-profile-by-short-code.js new file mode 100644 index 0000000000..a5c08eb36e --- /dev/null +++ b/server/src/routes/group-user-profile-by-short-code.js @@ -0,0 +1,18 @@ +const DB = require('../db.js') +const log = require('tangy-log').log + +module.exports = async (req, res) => { + try { + const groupDb = new DB(req.params.groupId) + const userProfileShortCode = req.params.userProfileShortCode + + const result = await groupDb.query("userProfileByUserProfileShortCode", { key: userProfileShortCode, limit: 1, include_docs: true }); + const profile = result.rows[0] + const data = profile ? {_id: profile.id, key: profile.id, formId: profile.doc.form.id, collection: profile.doc.collection} : undefined + res.send(data) + + } catch (error) { + log.error(error); + res.status(500).send(error); + } +} \ No newline at end of file diff --git a/server/src/scripts/generate-csv/batch.js b/server/src/scripts/generate-csv/batch.js index 0c5c0bdcc4..aed71dccba 100755 --- a/server/src/scripts/generate-csv/batch.js +++ b/server/src/scripts/generate-csv/batch.js @@ -27,7 +27,6 @@ function getData(dbName, formId, skip, batchSize, year, month) { try { const key = (year && month) ? `${formId}_${year}_${month}` : formId const target = `${dbDefaults.prefix}/${dbName}/_design/tangy-reporting/_view/resultsByGroupFormId?keys=["${key}"]&include_docs=true&skip=${skip}&limit=${limit}` - console.log(target) axios.get(target) .then(response => { resolve(response.data.rows.map(row => row.doc)) @@ -42,14 +41,56 @@ function getData(dbName, formId, skip, batchSize, year, month) { }); } +function handleCSVReplacementAndDisabledFields(value, csvReplacementCharacters) { + // Handle csv-safe character replacement and disabled fields + if (Array.isArray(value)) { + return '' + } + if (typeof value === 'string') { + if (csvReplacementCharacters) { + csvReplacementCharacters.forEach(expression => { + const search = expression["search"]; + const replace = expression["replace"]; + if (search && replace) { + const re = new RegExp(search, 'g') + try { + value = value.replace(re, replace) + } catch (e) { + console.log("ERROR! re: " + re + " replace: " + replace + " value: " + value + " Error: " + e) + } + } + }) + } + } + if (typeof header === 'string' && header.split('.').length === 3) { + const itemId = header.split('.')[1] + if (itemId && doc[`${itemId}_disabled`] === 'true') { + if (outputDisabledFieldsToCSV) { + return value + } else { + return process.env.T_REPORTING_MARK_SKIPPED_WITH + } + } else { + if (value === undefined) { + return process.env.T_REPORTING_MARK_UNDEFINED_WITH + } else { + return value + } + } + } else { + if (value === undefined) { + return process.env.T_REPORTING_MARK_UNDEFINED_WITH + } else { + return value + } + } +} + async function batch() { const state = JSON.parse(await readFile(params.statePath)) - console.log("state.skip: " + state.skip) const docs = await getData(state.dbName, state.formId, state.skip, state.batchSize, state.year, state.month) let outputDisabledFieldsToCSV = state.groupConfigurationDoc? state.groupConfigurationDoc["outputDisabledFieldsToCSV"] : false - console.log("outputDisabledFieldsToCSV: " + outputDisabledFieldsToCSV) let csvReplacementCharacters = state.groupConfigurationDoc? state.groupConfigurationDoc["csvReplacementCharacters"] : false - console.log("csvReplacementCharacters: " + JSON.stringify(csvReplacementCharacters)) // let csvReplacement = csvReplacementCharacters? JSON.parse(csvReplacementCharacters) : false if (docs.length === 0) { state.complete = true @@ -59,85 +100,49 @@ async function batch() { try { rows = [] docs.forEach(doc => { - let row = [doc._id, ...state.headersKeys.map(header => { - // Check to see if variable comes from a section that was disabled. - if (doc.type === 'attendance' && header === 'attendanceList') { - // skip - } else if (doc.type === 'scores' && header === 'scoreList') { - // skip - } else { - let value = doc[header]; - console.log("header: " + header + " value: " + value) - if (typeof value === 'string') { - if (csvReplacementCharacters) { - csvReplacementCharacters.forEach(expression => { - const search = expression["search"]; - const replace = expression["replace"]; - if (search && replace) { - const re = new RegExp(search, 'g') - try { - value = value.replace(re, replace) - } catch (e) { - console.log("ERROR! re: " + re + " replace: " + replace + " value: " + value + " Error: " + e) - } - } - }) - } - } - if (typeof header === 'string' && header.split('.').length === 3) { - console.log("Checking header: " + header + " to see if it is disabled.") - const itemId = header.split('.')[1] - if (itemId && doc[`${itemId}_disabled`] === 'true') { - if (outputDisabledFieldsToCSV) { - return value - } else { - return process.env.T_REPORTING_MARK_SKIPPED_WITH - } - } else { - if (value === undefined) { - return process.env.T_REPORTING_MARK_UNDEFINED_WITH - } else { - return value - } - } - } else { - if (value === undefined) { - return process.env.T_REPORTING_MARK_UNDEFINED_WITH + if (doc.type === 'attendance') { + doc.attendanceList.forEach(listItem => { + let row = ([doc._id, ...state.headersKeys.map(header => { + if (listItem[header]) { + return listItem[header]; } else { - return value + let value = doc[header]; + return handleCSVReplacementAndDisabledFields(value, csvReplacementCharacters); } - } - } - })] - if (doc.type === 'attendance') { - doc.attendanceList.forEach(attendance => { - let row = [doc._id, ...state.headersKeys.map(header => { - let value = attendance[header]; - console.log("header: " + header + " value: " + value) - return value - })] + })]) rows.push(row) }) } else if (doc.type === 'scores') { - doc.scoreList.forEach(score => { - let row = [doc._id, ...state.headersKeys.map(header => { - let value = score[header]; - console.log("header: " + header + " value: " + value) - return value - })] + doc.scoreList.forEach(listItem => { + let row = ([doc._id, ...state.headersKeys.map(header => { + if (listItem[header]) { + return listItem[header]; + } else { + let value = doc[header]; + return handleCSVReplacementAndDisabledFields(value, csvReplacementCharacters); + } + })]) rows.push(row) }) } else if (doc.type === 'behavior') { - doc.studentBehaviorList.forEach(behavior => { - let row = [doc._id, ...state.headersKeys.map(header => { - let value = behavior[header]; - console.log("header: " + header + " value: " + value) - return value - })] + doc.studentBehaviorList.forEach(listItem => { + let row = ([doc._id, ...state.headersKeys.map(header => { + if (listItem[header]) { + return listItem[header]; + } else { + let value = doc[header]; + return handleCSVReplacementAndDisabledFields(value, csvReplacementCharacters); + } + })]) rows.push(row) }) } else { - // rows = docs.map(doc => { + let row = [doc._id, + ...state.headersKeys.map(header => { + let value = doc[header]; + return handleCSVReplacementAndDisabledFields(value, csvReplacementCharacters); + }) + ] rows.push(row) } }) diff --git a/server/src/scripts/import-archives-v2/bin.js b/server/src/scripts/import-archives-v2/bin.js new file mode 100755 index 0000000000..3f75e2ea8b --- /dev/null +++ b/server/src/scripts/import-archives-v2/bin.js @@ -0,0 +1,55 @@ +#!/usr/bin/env node +if (!process.argv[2]) { + console.log('Place archives from clients into the ./data/archives folder on the host machine then run...') + console.log(' ./bin.js ') + process.exit() +} + +const util = require('util') +const readdir = util.promisify(require('fs').readdir) +const readFile = util.promisify(require('fs').readFile) +const pako = require('pako') +const axios = require('axios') +const url = `http://localhost/api/${process.argv[2]}/upload` +const ARCHIVES_PATH = '/archives' + + +async function go() { + const archivesList = await readdir(ARCHIVES_PATH) + for (const archivePath of archivesList) { + const archiveContents = await readFile(`${ARCHIVES_PATH}/${archivePath}`, 'utf-8') + const docsArray = ([...JSON.parse(archiveContents)]).find(item=>item.docs)?.docs + const userProfileDoc = docsArray.find(item=> item.form&& item.form.id=== 'user-profile') + console.log(userProfileDoc) + const docs = docsArray + .map(item => { + if (item.collection !== 'TangyFormResponse') return + if (item.form && item.form.id !== 'user-profile') { + item.items[0].inputs.push({ + name: 'userProfileId', + value: userProfileDoc._id + }) + } + return item + }) + .filter(doc => doc !== undefined) + for (const doc of docs) { + let body = pako.deflate(JSON.stringify({ doc }), {to: 'string'}) + await axios({ + method: 'post', + url, + data: `${body}`, + headers: { + 'content-type': 'text/plain', + 'Authorization': `${process.env.T_UPLOAD_TOKEN}` + } + }) + } + } +} + +try { + go() +} catch(e) { + console.log(e) +} diff --git a/server/src/scripts/info.sh b/server/src/scripts/info.sh index 6ebf79f85a..f36cb3b492 100755 --- a/server/src/scripts/info.sh +++ b/server/src/scripts/info.sh @@ -14,7 +14,8 @@ echo "generate-cases (Generate cases with group's case-export.json as echo "reset-all-devices (Reset server tokens and device keys for all devices, requires reinstall and set up on all devices after)" echo "push-all-groups-views (Push all database views into all groups)" echo "index-all-groups-views (Index all database views into all groups)" -echo "import-archives (Import client archives from the ./data/archives folder)" +echo "import-archives (Import client archives from the ./data/archives folder using old format)" +echo "import-archives-v2 (Import client archives from the ./data/archives folder using new format)" echo "release-apk (Release a Group App as an APK)" echo "release-pwa (Release a Group App as a PWA)" echo "release-dat (Release a Group APP as a Dat Archive)" diff --git a/server/src/scripts/reporting-worker-unpause.sh b/server/src/scripts/reporting-worker-unpause.sh index bcda35be9b..203fcb136c 100755 --- a/server/src/scripts/reporting-worker-unpause.sh +++ b/server/src/scripts/reporting-worker-unpause.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + if [ "$2" = "--help" ]; then echo "Usage:" echo " reporting-worker-pause" diff --git a/server/src/shared/services/group-responses/group-responses.service.ts b/server/src/shared/services/group-responses/group-responses.service.ts index 01531fd362..f7bbdf1cdf 100644 --- a/server/src/shared/services/group-responses/group-responses.service.ts +++ b/server/src/shared/services/group-responses/group-responses.service.ts @@ -137,11 +137,13 @@ export class GroupResponsesService { } async update(groupId, response) { + const tangerineModifiedOn = Date.now() try { const groupDb = this.getGroupsDb(groupId) const originalResponse = await groupDb.get(response._id) await groupDb.put({ ...response, + tangerineModifiedOn, _rev: originalResponse._rev }) const freshResponse = await groupDb.get(response._id) @@ -149,7 +151,7 @@ export class GroupResponsesService { } catch (e) { try { const groupDb = this.getGroupsDb(groupId) - await groupDb.put(response) + await groupDb.put({...response, tangerineModifiedOn}) const freshResponse = await groupDb.get(response._id) return freshResponse } catch (e) { diff --git a/server/src/upgrade/v3.31.0.js b/server/src/upgrade/v3.31.0.js new file mode 100755 index 0000000000..0ec27efb8e --- /dev/null +++ b/server/src/upgrade/v3.31.0.js @@ -0,0 +1,13 @@ +#!/usr/bin/env node +const util = require("util"); +const exec = util.promisify(require('child_process').exec) + +async function go() { + console.log('Updating views with a new view used for the User Profile listing.') + try { + await exec(`/tangerine/server/src/scripts/push-all-groups-views.js `) + } catch (e) { + console.log(e) + } +} +go() diff --git a/tangerine-preview/package.json b/tangerine-preview/package.json index a5cd8783fc..d99cae3438 100644 --- a/tangerine-preview/package.json +++ b/tangerine-preview/package.json @@ -1,6 +1,6 @@ { "name": "tangerine-preview", - "version": "3.12.5", + "version": "3.31.0", "description": "", "main": "index.js", "bin": {