diff --git a/backend/db/models/study_step.js b/backend/db/models/study_step.js index 6016a9775..fac20c274 100644 --- a/backend/db/models/study_step.js +++ b/backend/db/models/study_step.js @@ -93,6 +93,38 @@ module.exports = (sequelize, DataTypes) => { return await this.findAll({where: {documentId}}); } + /** + * Get the study steps of a study, sorted by their order. + * + * A soft-deleted step breaks the studyStepPrevious chain (getAllByKey excludes it, so the + * walk can't find the step that points past it). Any steps left unreachable this way are + * still non-deleted data and are appended at the end (id order) rather than dropped, so an + * export or session-progression check never silently loses a live step. + * @param studyId + * @returns {Promise<[]>} Array of study step objects + */ + static async getSortedStudySteps(studyId) { + const studySteps = await sequelize.models.study_step.getAllByKey("studyId", studyId); + const studyStepsSorted = []; + let current = studySteps.find(step => step.studyStepPrevious === null); + + while (current) { + studyStepsSorted.push(current); + current = studySteps.find(step => step.studyStepPrevious === current.id); + } + + if (studyStepsSorted.length < studySteps.length) { + const includedIds = new Set(studyStepsSorted.map(step => step.id)); + const orphanedSteps = studySteps + .filter(step => !includedIds.has(step.id)) + .sort((a, b) => a.id - b.id); + console.warn(`getSortedStudySteps: study ${studyId} has ${orphanedSteps.length} step(s) unreachable via studyStepPrevious (likely a soft-deleted step broke the chain); appending them out of order instead of dropping them.`); + studyStepsSorted.push(...orphanedSteps); + } + + return studyStepsSorted; + } + /** * Adding a new study step * @param data diff --git a/backend/package-lock.json b/backend/package-lock.json index 378bac5b4..3b9528c5a 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -70,7 +70,7 @@ "license": "Apache-2.0", "devDependencies": { "cross-env": "^7.0.3", - "jest": "^30.5.0" + "jest": "^30.5.1" } }, "../utils/modules/editor-delta-conversion": { @@ -78,11 +78,12 @@ "license": "Apache-2.0", "dependencies": { "quill": "2.0.3", - "quill-delta": "^5.1.0" + "quill-delta": "^5.1.0", + "quill-delta-to-html": "0.12.1" }, "devDependencies": { "cross-env": "^7.0.3", - "jest": "^30.5.0" + "jest": "^30.5.1" } }, "node_modules/@babel/code-frame": { @@ -116,6 +117,7 @@ "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -601,29 +603,6 @@ "kuler": "^2.0.0" } }, - "node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@emnapi/wasi-threads": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", @@ -2610,6 +2589,7 @@ "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==", "license": "MIT", + "peer": true, "dependencies": { "follow-redirects": "^1.16.0", "form-data": "^4.0.6", @@ -2771,6 +2751,7 @@ "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.2.tgz", "integrity": "sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ==", "license": "Apache-2.0", + "peer": true, "peerDependencies": { "bare-abort-controller": "*" }, @@ -3032,6 +3013,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.11.20", "caniuse-lite": "^1.0.30001810", @@ -4212,6 +4194,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -7075,6 +7058,7 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.0.tgz", "integrity": "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0", @@ -7115,7 +7099,6 @@ "resolved": "https://registry.npmjs.org/pg-cursor/-/pg-cursor-2.22.0.tgz", "integrity": "sha512-knzXLKqarTjOvb3qDSW0JiGsazmxwEKXrqHfWRte7XUsOYccQRafn3BLnQobWwInkzFJSyOej8y8cQRh2z3kGw==", "license": "MIT", - "peer": true, "peerDependencies": { "pg": "^8" } @@ -7693,6 +7676,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "@types/debug": "^4.1.8", "@types/validator": "^13.7.17", @@ -9078,6 +9062,7 @@ "resolved": "https://registry.npmjs.org/winston/-/winston-3.19.0.tgz", "integrity": "sha512-LZNJgPzfKR+/J3cHkxcpHKpKKvGfDZVPS4hfJCc4cCG0CgYzvlD6yE/S3CIL/Yt91ak327YCpiF/0MyeZHEHKA==", "license": "MIT", + "peer": true, "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.8", diff --git a/backend/utils/helper/export.js b/backend/utils/helper/export.js new file mode 100644 index 000000000..c1bb51f0f --- /dev/null +++ b/backend/utils/helper/export.js @@ -0,0 +1,493 @@ +const fs = require('fs'); +const { faker } = require('@faker-js/faker'); +const JSZip = require('jszip'); +const { deriveUserSeed } = require('../../webserver/auth/utils'); +const path = require('path'); +const storageDir = path.join(__dirname, "..", "..", "..", "files"); +const Papa = require('papaparse'); +const { Readable } = require('stream'); + +const SUPPORTED_EXPORT_TYPES = new Set(["submissions", "grades", "documents", "studies", "userBehaviour"]); + + +/** + * Validates an export request and loads the project/users it targets. + * @param {Object} server - The server instance providing database models and Sequelize operators. + * @param {Object} params - Validation inputs. + * @param {number} params.parsedProjectId - The numeric project id. + * @param {string} params.exportType - The requested export type. + * @param {string} params.normalizedGradeFormat - The requested grade format, lowercased. + * @param {Array} params.userIds - Parsed user ids to export. + * @param {*} params.workflowIds - Raw workflowIds value from the request body, parsed here. + * @param {number} params.currentUserId - The id of the user making the export request. + * @returns {Promise<{success: boolean, status?: number, message?: string, users?: Array, workflowIds?: Array}>} + */ +async function loadExportRequestContext(server, { parsedProjectId, exportType, normalizedGradeFormat, userIds, workflowIds, currentUserId }) { + if (!Number.isInteger(parsedProjectId)) { + return { success: false, status: 400, message: "Missing projectId." }; + } + if (!SUPPORTED_EXPORT_TYPES.has(exportType)) { + return { success: false, status: 400, message: "Unsupported export type." }; + } + try { + workflowIds = typeof workflowIds === 'string' ? JSON.parse(workflowIds) : workflowIds; + } catch (e) { + server.logger.warn("Could not parse workflowIds:", workflowIds); + workflowIds = []; + } + if (!Array.isArray(workflowIds)) workflowIds = []; + if (exportType === "studies" && workflowIds.length === 0) { + return { success: false, status: 400, message: "No workflows selected." }; + } + if (exportType === "grades" && !["json", "csv"].includes(normalizedGradeFormat)) { + return { success: false, status: 400, message: "Unsupported grade format. Use json or csv." }; + } + if (userIds.length === 0) { + server.logger.warn("Export aborted: No valid users selected."); + return { success: false, status: 400, message: "No valid users selected." }; + } + + const project = await server.db.models.project.findOne({ where: { id: parsedProjectId } }); + if (!project) { + server.logger.warn(`${parsedProjectId} does not exist.`); + return { success: false, status: 403, message: "The selected project does not exist." }; + } + + const isAdmin = await resolveIsAdmin(server, currentUserId); + const isSelfOnlyExport = userIds.every(id => Number(id) === Number(currentUserId)); + if (!isAdmin && project.userId !== currentUserId && !isSelfOnlyExport) { + server.logger.warn(`User ${currentUserId} attempted to export project ${parsedProjectId} without access.`); + return { success: false, status: 403, message: "You don't have access to this project." }; + } + + const { Op } = server.db.Sequelize; + const users = await server.db.models.user.findAll({ where: { id: { [Op.in]: userIds } } }); + if (users.length === 0) { + server.logger.warn("Export aborted: No existing users to export."); + return { success: false, status: 400, message: "No authorized users to export." }; + } + + return { success: true, users, workflowIds }; +} + +/** + * Opens a zip file, replaces the student's real name with a fake name in all .tex files, + * and returns the modified zip as a Buffer. + * @param {string} filePath - Path to the original zip file on disk + * @param {string} realName - The student's real name to search for + * @param {string} fakeName - The generated fake name to insert + * @returns {Promise} - The newly generated zip file buffer + */ +async function replaceAuthorInZip(filePath, realName, fakeName) { + const fileData = fs.readFileSync(filePath); + const zip = await JSZip.loadAsync(fileData); + const getFirstAndLastNameTokens = (name) => { + const parts = String(name || "").trim().split(/\s+/).filter(Boolean); + if (parts.length === 0) return ["", ""]; + if (parts.length === 1) return [parts[0], ""]; + return [parts[0], parts[parts.length - 1]]; + }; + const [realFirstName, realLastName] = getFirstAndLastNameTokens(realName); + const [fakeFirstName, fakeLastName] = getFirstAndLastNameTokens(fakeName); + + const authorRegex = /\\author\s*\{[^}]*\}/g; + + for (const [relativePath, zipEntry] of Object.entries(zip.files)) { + if (!zipEntry.dir && relativePath.toLowerCase().endsWith('.tex')) { + let text = await zipEntry.async("string"); + text = text.replace(authorRegex, `\\author{${fakeName}}`); + if (realFirstName && fakeFirstName) text = text.split(realFirstName).join(fakeFirstName); + if (realLastName && fakeLastName) text = text.split(realLastName).join(fakeLastName); + + zip.file(relativePath, text); + } + } + + return await zip.generateAsync({ + type: "nodebuffer", + compression: "DEFLATE" + }); +} + +/** + * Constructs a mapping of user IDs to aliases and generates a + * corresponding CSV string. + * @param {Array} users - Array of user objects from the database. + * @param {boolean} shouldGenerateAliases - Whether the export should use fake names. + * @param {boolean} hasPrivateInfoRight - Whether the current user is allowed to see/export full names. + * @param {number|string} fakerSeed - The base integer seed (from the form input). + * @param {string} salt - The hex-encoded salt string from the user's database record. + * @returns {Object} An object containing: + * - userMapping: An object mapping user IDs to their generated fake names. + * - mappingCsv: A CSV-formatted string containing the mapping (conditionally includes real names). + */ +function buildUserMapping(users, shouldGenerateAliases, hasPrivateInfoRight, fakerSeed, salt) { + let userMapping = {}; + let csvRows = []; + + if (shouldGenerateAliases) { + if (fakerSeed !== null && fakerSeed !== undefined && fakerSeed !== "" && !isNaN(parseInt(fakerSeed, 10))) { + const derivedFakerSeed = deriveUserSeed(parseInt(fakerSeed, 10), salt); + faker.seed(derivedFakerSeed); + } + + const sortedUsers = [...users].sort((a, b) => Number(a.id) - Number(b.id)); + sortedUsers.forEach(u => { + const realUsername = u.userName; + const realName = `${u.firstName} ${u.lastName}`; + const fakeName = `${faker.person.firstName()} ${faker.person.lastName()}`; + + userMapping[u.id] = fakeName; + + let rowData = { + "Username": realUsername + }; + if (hasPrivateInfoRight) { + rowData["Real Name"] = realName; + } + + rowData["Generated Alias"] = fakeName; + + csvRows.push(rowData); + }); + } + const mappingCsv = csvRows.length > 0 ? Papa.unparse(csvRows) : ""; + return { userMapping, mappingCsv }; +} + +/** + * Normalizes a folder name so it can be used as a ZIP path segment without + * accidentally introducing invalid filename characters or nested paths. + * + * @param {string|number|null|undefined} value - The raw folder name. + * @returns {string} A sanitized folder name with reserved characters replaced. + */ +function sanitizeFolderName(value) { + return String(value || "unknown") + .replace(/[<>:"/\\|?*\x00-\x1F]/g, "_") + .replace(/\s+/g, " ") + .trim(); +} + +/** + * Returns a user's display name based on private info permissions. + * + * @param {Object|null} user - The user record. + * @param {boolean} hasPrivateInfoRight - Whether real names are allowed. + * @returns {string|null} Full name or username depending on permissions. + */ +function getPrivateAwareName(user, hasPrivateInfoRight) { + if (!user) return null; + if (hasPrivateInfoRight) return `${user.firstName} ${user.lastName}`.trim(); + // Usernames are considered anonymous-enough for exports when real names are restricted. + return user.userName ?? null; +} + +/** + * Resolves the display name for a user based on the current export settings. + * This wraps getPrivateAwareName with alias support for anonymized exports. + * + * @param {Object} user - The user record to display. + * @param {boolean} shouldGenerateAliases - Whether aliases should replace real names. + * @param {boolean} hasPrivateInfoRight - Whether the current user may export real names. + * @param {Object} userMapping - Map of user IDs to generated aliases. + * @returns {string} The display name to write into the export. + */ +function getDisplayName(user, shouldGenerateAliases, hasPrivateInfoRight, userMapping) { + if (shouldGenerateAliases) return userMapping[user.id]; + return getPrivateAwareName(user, hasPrivateInfoRight); +} + +/** + * Calculates the version number of a submission by traversing backwards + * through the chain of previous submissions. + * @param {Object} submission - The current submission object to start from. + * @param {Map} submissionMap - A Map containing all related + * submissions for quick lookup by ID. + * @returns {number} - The calculated version number (starting at 1 for the original). + */ +function calculateSubmissionVersion(submission, submissionMap) { + let version = 1; + let currentSub = submission; + while (currentSub && currentSub.previousSubmissionId) { + const prevSub = submissionMap.get(currentSub.previousSubmissionId); + if (!prevSub) break; + version++; + currentSub = prevSub; + } + return version; +} + +/** + * Resolves which of the given user ids have opted into data sharing. + * + * @param {Object} server - The server instance providing database models. + * @param {Array} candidateUserIds - User ids to check consent for. + * @returns {Promise>} Set of user ids that accepted data sharing. + */ +async function getConsentedUserIds(server, candidateUserIds) { + if (candidateUserIds.length === 0) return new Set(); + const consentedUsers = await server.db.models.user.findAll({ + where: { id: candidateUserIds }, + attributes: ['id', 'acceptDataSharing'], + raw: true, + }); + return new Set(consentedUsers.filter(u => u.acceptDataSharing).map(u => u.id)); +} + +/** + * Appends a stored file (by hash + extension) to the archive if it exists on disk, otherwise warns. + * @param {Object} server - The server instance providing the logger. + * @param {Object} archive - The archiver instance to append the file to. + * @param {string} hash - The document's storage hash. + * @param {string} extension - File extension including the dot, e.g. ".pdf". + * @param {string} archivePath - Destination path inside the ZIP archive. + * @param {string} typeLabel - Human-readable type label used in the warning log. + * @returns {void} + */ +function appendStoredFileIfExists(server, archive, hash, extension, archivePath, typeLabel) { + const filePath = path.join(storageDir, `${hash}${extension}`); + if (fs.existsSync(filePath)) { + archive.file(filePath, { name: archivePath }); + } else { + server.logger.warn(`[DocumentExport] ${typeLabel} not found for document ${hash}`); + } +} + +/** + * Appends a stored ZIP document to the archive, replacing the owner's real name with their + * alias in any .tex file inside it when aliases are requested. Falls back to a plain copy if + * the file is missing, aliasing isn't requested, the owner is unknown, or anonymization fails. + * @param {Object} server - The server instance providing the logger. + * @param {Object} archive - The archiver instance to append the file to. + * @param {string} hash - The document's storage hash. + * @param {string} archivePath - Destination path inside the ZIP archive. + * @param {boolean} shouldGenerateAliases - Whether the export should anonymize author names. + * @param {Object} userMapping - Map of user IDs to generated aliases. + * @param {Object|null} ownerUser - The document owner's user record (for the real name to replace). + * @param {Map|null} [anonymizedBufferCache] - Optional cache, keyed by hash, so a + * document appended under several archive paths (e.g. one per review session) only pays for the + * unzip/substitute/rezip round-trip once. + * @returns {Promise} + */ +async function appendZipFileAnonymized(server, archive, hash, archivePath, shouldGenerateAliases, userMapping, ownerUser, anonymizedBufferCache = null) { + if (anonymizedBufferCache?.has(hash)) { + archive.append(anonymizedBufferCache.get(hash), { name: archivePath }); + return; + } + + const filePath = path.join(storageDir, `${hash}.zip`); + if (!fs.existsSync(filePath)) { + server.logger.warn(`[DocumentExport] ZIP not found for document ${hash}`); + return; + } + if (shouldGenerateAliases && ownerUser) { + const realName = `${ownerUser.firstName || ""} ${ownerUser.lastName || ""}`.trim(); + const fakeName = userMapping[ownerUser.id]; + try { + const newZipBuffer = await replaceAuthorInZip(filePath, realName, fakeName); + anonymizedBufferCache?.set(hash, newZipBuffer); + archive.append(newZipBuffer, { name: archivePath }); + return; + } catch (err) { + server.logger.error(`Failed to change names for zip ${hash}:`, err); + } + } + archive.file(filePath, { name: archivePath }); +} + +/** + * Resolves whether a user may see other users' full names in exports (admins always can). + * @param {Object} server - The server instance providing database models. + * @param {number} userId - The requesting user's id. + * @returns {Promise} Whether the user has the private-info export right. + */ +async function resolveHasPrivateInfoRight(server, userId) { + const roleIds = await server.db.models["user_role_matching"].getUserRolesById(userId); + const isAdmin = await server.db.models["user_role_matching"].isAdminInUserRoles(roleIds); + if (isAdmin) return true; + + const userRightsObj = await server.db.models.user.getUserRights(userId); + if (!userRightsObj) return false; + + const allRights = Object.values(userRightsObj).flat(); + return allRights.includes('frontend.dashboard.studies.view.userPrivateInfo'); +} + +/** + * Parses the raw userIds field from a request body into an array, tolerating a JSON-encoded string. + * @param {Object} server - The server instance providing the logger. + * @param {*} rawUserIds - The raw value from req.body.userIds. + * @returns {Array} Parsed array of user ids, or an empty array if parsing fails. + */ +function parseUserIds(server, rawUserIds) { + try { + const parsed = typeof rawUserIds === 'string' ? JSON.parse(rawUserIds) : rawUserIds; + return Array.isArray(parsed) ? parsed : []; + } catch (e) { + server.logger.warn("Could not parse userIds:", rawUserIds); + return []; + } +} + +/** + * Resolves each annotation's tagId to its tag name and attaches it as `tagName`. + * @param {Object} server - The server instance providing database models. + * @param {Array} annotations - Annotation records, each optionally carrying a `tagId`. + * @returns {Promise>} The annotations with a `tagName` field added (null if untagged). + */ +async function attachTagNames(server, annotations) { + const tagIds = [...new Set(annotations.map(a => a.tagId).filter(Boolean))]; + if (tagIds.length === 0) return annotations; + + const tags = await server.db.models.tag.findAll({ + where: { id: tagIds }, + attributes: ['id', 'name'], + raw: true, + }); + const tagNameById = new Map(tags.map(t => [t.id, t.name])); + + return annotations.map(a => ({ ...a, tagName: tagNameById.get(a.tagId) ?? null })); +} + +/** + * Resolves whether a user holds an admin role. + * @param {Object} server - The server instance providing database models. + * @param {number} userId - The user's id. + * @returns {Promise} Whether the user is an admin. + */ +async function resolveIsAdmin(server, userId) { + const roleIds = await server.db.models["user_role_matching"].getUserRolesById(userId); + return await server.db.models["user_role_matching"].isAdminInUserRoles(roleIds); +} + +/** + * Builds a Readable that emits a JSON array incrementally, paging through fetchPage(lastId, limit) + * using keyset pagination so the full result set is never held in memory at once. + * @param {(lastId: number, limit: number) => Promise>} fetchPage - Fetches the next page of rows after lastId, ordered by id. + * @param {(row: Object) => Object} [mapRow] - Transforms each row before serialization. Defaults to the identity function. + * @param {number} [pageSize=1000] - Number of rows to fetch per page. + * @returns {Readable} A stream emitting the JSON-serialized array. + */ +function createJsonArrayStream(fetchPage, mapRow = (row) => row, pageSize = 1000) { + let lastId = 0; + let started = false; + let finished = false; + let isFirst = true; + let fetching = false; + + return new Readable({ + read() { + if (fetching || finished) return; + fetching = true; + + (async () => { + try { + if (!started) { + this.push('['); + started = true; + } + + const rows = await fetchPage(lastId, pageSize); + if (rows.length === 0) { + this.push('\n]'); + this.push(null); + finished = true; + return; + } + + let chunk = ''; + for (const row of rows) { + chunk += (isFirst ? '' : ',') + '\n' + JSON.stringify(mapRow(row)); + isFirst = false; + } + lastId = rows[rows.length - 1].id; + + if (rows.length < pageSize) { + chunk += '\n]'; + this.push(chunk); + this.push(null); + finished = true; + } else { + this.push(chunk); + } + } catch (err) { + this.destroy(err); + } finally { + fetching = false; + } + })(); + } + }); +} + +/** + * Builds a Readable that emits CSV rows incrementally, paging through fetchPage(lastId, limit) + * using keyset pagination so the full result set is never held in memory at once. + * @param {(lastId: number, limit: number) => Promise>} fetchPage - Fetches the next page of rows after lastId, ordered by id. + * @param {(row: Object) => Object} mapRow - Transforms each row into the record written to CSV. + * @param {Array} fields - Column names/order for the CSV header. + * @param {number} [pageSize=1000] - Number of rows to fetch per page. + * @returns {Readable} A stream emitting CSV text, with the header written once on the first non-empty chunk. + */ +function createCsvRowsStream(fetchPage, mapRow, fields, pageSize = 1000) { + let lastId = 0; + let started = false; + let finished = false; + let fetching = false; + + return new Readable({ + read() { + if (fetching || finished) return; + fetching = true; + + (async () => { + try { + const rows = await fetchPage(lastId, pageSize); + if (rows.length === 0) { + if (!started) this.push(Papa.unparse({ fields, data: [] })); + this.push(null); + finished = true; + return; + } + + const records = rows.map(mapRow); + const csvChunk = Papa.unparse(records, { header: !started }) + '\n'; + started = true; + lastId = rows[rows.length - 1].id; + + this.push(csvChunk); + + if (rows.length < pageSize) { + this.push(null); + finished = true; + } + } catch (err) { + this.destroy(err); + } finally { + fetching = false; + } + })(); + } + }); +} + +module.exports = { + replaceAuthorInZip, + buildUserMapping, + sanitizeFolderName, + getPrivateAwareName, + getDisplayName, + calculateSubmissionVersion, + getConsentedUserIds, + appendStoredFileIfExists, + appendZipFileAnonymized, + resolveHasPrivateInfoRight, + parseUserIds, + loadExportRequestContext, + SUPPORTED_EXPORT_TYPES, + attachTagNames, + resolveIsAdmin, + createJsonArrayStream, + createCsvRowsStream +}; \ No newline at end of file diff --git a/backend/utils/helper/exportGrades.js b/backend/utils/helper/exportGrades.js new file mode 100644 index 000000000..c2c0871b4 --- /dev/null +++ b/backend/utils/helper/exportGrades.js @@ -0,0 +1,330 @@ +const { calculateAssessmentScore, buildScoresFromState } = require('assessment-score'); +const { getDisplayName, getPrivateAwareName } = require('./export.js'); + +const ASSESSMENT_RESULT_KEY = "assessment_result"; + +/** + * Parses an assessment state payload when it is stored as JSON text. + * + * @param {Object} server - The server instance providing the logger. + * @param {string} rawAssessmentState - The raw JSON string from document_data. + * @returns {Object} The parsed assessment state or an empty object on failure. + */ +function parseAssessmentState(server, rawAssessmentState) { + try { + const parsed = JSON.parse(rawAssessmentState); + return parsed && typeof parsed === "object" ? parsed : {}; + } catch (error) { + server.logger.warn("Failed to parse assessment state:", error.message); + return {}; + } +} + +/** + * Reads the rubric configuration id from a study step configuration payload. + * + * @param {Object|null|undefined} studyStepConfiguration - The study step's configuration object. + * @returns {number|null} The referenced configuration id or null when unavailable. + */ +function getAssessmentConfigurationId(studyStepConfiguration) { + if (!studyStepConfiguration || typeof studyStepConfiguration !== "object") return null; + const rawId = + studyStepConfiguration.settings?.configurationId ?? + studyStepConfiguration.configurationId ?? + null; + const parsedId = Number(rawId); + return Number.isInteger(parsedId) ? parsedId : null; +} + +/** + * Resolves the assessment rubric configuration referenced by a study step. + * Study steps are expected to store only a configurationId; rubric content + * is loaded from the configuration table. + * + * @param {Object|null|undefined} studyStepConfiguration - The study step's configuration JSON. + * @param {Map} configurationsById - Loaded configuration records by id. + * @returns {Object|null} Assessment config content (with rubrics) or null. + */ +function resolveAssessmentConfigurationContent(studyStepConfiguration, configurationsById) { + const configurationId = getAssessmentConfigurationId(studyStepConfiguration); + if (configurationId === null) return null; + + const configuration = configurationsById.get(configurationId); + return configuration?.content ?? null; +} + +/** + * Records the assessment configuration content for a given configuration id, + * so each distinct configuration used across a grade export gets its own + * criteria reference file (a grade export may now span multiple configurations). + * + * @param {Map} referencesByConfigId - Mutable map of configurationId -> reference content. + * @param {number|null} configurationId - Resolved persisted configuration id. + * @param {Object|null} assessmentConfig - Resolved assessment configuration content. + * @returns {void} + */ +function addCriteriaReferenceEntry(referencesByConfigId, configurationId, assessmentConfig) { + if (!assessmentConfig || typeof assessmentConfig !== "object") return; + if (!Number.isInteger(configurationId)) return; + if (referencesByConfigId.has(configurationId)) return; + + referencesByConfigId.set(configurationId, { + configurationId, + ...assessmentConfig + }); +} + +/** + * Builds a flat CSV row for a grade export record. + * The row contains backend export metadata columns followed by + * one column per assessment criterion score. + * + * @param {Object} record - Prepared grade export record. + * @returns {Object} A flat object suitable for Papa.unparse. + */ +function buildGradeCsvRow(record) { + const criterionScores = record.scores && typeof record.scores === "object" ? record.scores : {}; + return { + projectId: record.projectId, + userId: record.userId, + userExtId: record.userExtId, + userName: record.userName, + displayName: record.displayName, + submissionId: record.submissionId, + submissionExtId: record.submissionExtId, + studySessionId: record.studySessionId, + studyName: record.studyName, + studyStepId: record.studyStepId, + studyStepType: record.studyStepType, + configurationId: record.configurationId, + studyOwner: record.studyOwner, + sessionOwner: record.sessionOwner, + author: record.author, + totalPoints: record.totalPoints, + createdAt: record.createdAt, + ...criterionScores + }; +} + +/** + * Loads the related entities needed to turn raw assessment_result rows into + * export-ready grade records. + * + * @param {Object} server - The server instance with Sequelize models. + * @param {Array} gradeRows - Assessment result rows with attached documents. + * @param {Array} users - The selected document owners for the export. + * @returns {Promise} Lookup maps for related grade-export entities. + */ +async function loadGradeExportContext(server, gradeRows, users) { + const { Op } = server.db.Sequelize; + + const sessionIds = [...new Set(gradeRows.map((row) => row.studySessionId).filter(Boolean))]; + const studySessions = sessionIds.length > 0 + ? await server.db.models.study_session.findAll({ + where: { id: { [Op.in]: sessionIds }, deleted: false }, + raw: true + }) + : []; + const sessionsById = new Map(studySessions.map((session) => [session.id, session])); + + const studyIds = [...new Set(studySessions.map((session) => session.studyId).filter(Boolean))]; + const studies = studyIds.length > 0 + ? await server.db.models.study.findAll({ + where: { id: { [Op.in]: studyIds }, deleted: false }, + raw: true + }) + : []; + const studiesById = new Map(studies.map((study) => [study.id, study])); + + const studyStepIds = [...new Set(gradeRows.map((row) => row.studyStepId).filter(Boolean))]; + const studySteps = studyStepIds.length > 0 + ? await server.db.models.study_step.findAll({ + where: { id: { [Op.in]: studyStepIds }, deleted: false }, + raw: true + }) + : []; + const studyStepsById = new Map(studySteps.map((studyStep) => [studyStep.id, studyStep])); + + const configurationIds = [...new Set( + studySteps + .map((studyStep) => getAssessmentConfigurationId(studyStep.configuration)) + .filter((id) => id !== null) + )]; + const configurations = configurationIds.length > 0 + ? await server.db.models.configuration.findAll({ + where: { id: { [Op.in]: configurationIds }, deleted: false }, + raw: true + }) + : []; + const configurationsById = new Map(configurations.map((configuration) => [configuration.id, configuration])); + + // The export references study/session owners in addition to the selected document owners. + const relatedUserIds = [...new Set([ + ...users.map((user) => user.id), + ...studySessions.map((session) => session.userId), + ...studies.map((study) => study.userId) + ].filter(Boolean))]; + const relatedUsers = relatedUserIds.length > 0 + ? await server.db.models.user.findAll({ where: { id: { [Op.in]: relatedUserIds } }, raw: true }) + : []; + const usersById = new Map(relatedUsers.map((user) => [user.id, user])); + + return { + sessionsById, + studiesById, + studyStepsById, + configurationsById, + usersById + }; +} + +/** + * Orders grade records by session, then step within the session, then creation time. + * @param {Object} a - First grade record to compare. + * @param {Object} b - Second grade record to compare. + * @returns {number} Standard comparator result for Array#sort. + */ +function compareGradeRecords(a, b) { + const createdA = a.createdAt ? new Date(a.createdAt).getTime() : 0; + const createdB = b.createdAt ? new Date(b.createdAt).getTime() : 0; + return ( + (a.studySessionId || 0) - (b.studySessionId || 0) || + (a.studyStepId || 0) - (b.studyStepId || 0) || + createdA - createdB + ); +} + +/** + * Builds flat grade records for the given users/project by resolving each + * assessment_result row's session/study/step/configuration context and score. + * Shared by processGradesExport (grouped by user for JSON/CSV output) and + * processStudyBasedExport (grouped by session for a per-session scores.json). + * + * Pass `options.sessionIds` to scope the lookup to a set of study sessions. Without it the lookup + * is scoped by the owner of the assessed document, which only works when that owner is also the + * person the grades are being collected for. That holds for exposé assessments, where the assessed + * document belongs to the study owner, but not for review assessments: there the study belongs to + * the reviewer being assessed while the assessed review document belongs to the reviewed author, + * so an owner-scoped lookup returns nothing and the review grades are silently dropped. + * + * @param {Object} server - The server instance providing database models and Sequelize operators. + * @param {number} projectId - The project whose grades should be resolved. + * @param {Array} userIds - The selected document owners. + * @param {Array} users - Full user records for the selected users. + * @param {boolean} shouldGenerateAliases - Whether student names should be anonymized. + * @param {boolean} hasPrivateInfoRight - Whether the requester may export real names. + * @param {Object} userMapping - Map of user IDs to generated aliases. + * @returns {Promise<{records: Array, criteriaReferencesByConfigId: Map}>} + */ +async function buildGradeRecords(server, projectId, userIds, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, options = {}) { + const { Op } = server.db.Sequelize; + const { sessionIds = null } = options; + + const documentWhere = { projectId, deleted: false }; + if (!sessionIds) documentWhere.userId = { [Op.in]: userIds }; + + const gradeRows = await server.db.models.document_data.findAll({ + where: { + key: ASSESSMENT_RESULT_KEY, + deleted: false, + studySessionId: sessionIds ? { [Op.in]: sessionIds } : { [Op.ne]: null } + }, + include: [{ + model: server.db.models.document, + as: "document", + required: true, + where: documentWhere, + include: [{ + model: server.db.models.submission, + as: "submission", + required: false + }] + }], + order: [["studySessionId", "ASC"], ["studyStepId", "ASC"], ["createdAt", "ASC"]] + }); + + const { + sessionsById, + studiesById, + studyStepsById, + configurationsById, + usersById + } = await loadGradeExportContext(server, gradeRows, users); + + const records = []; + const criteriaReferencesByConfigId = new Map(); + for (const row of gradeRows) { + const document = row.document; + const ownerUser = usersById.get(document.userId); + if (!ownerUser) { + server.logger.warn("Skipping grade export row because the document owner could not be resolved.", { + documentId: document.id, + documentUserId: document.userId, + studySessionId: row.studySessionId, + studyStepId: row.studyStepId + }); + continue; + } + const session = sessionsById.get(row.studySessionId); + const reviewerUser = session ? usersById.get(session.userId) : null; + const study = session ? studiesById.get(session.studyId) : null; + const graderUser = study ? usersById.get(study.userId) : null; + const studyStep = studyStepsById.get(row.studyStepId); + const submission = document.submission; + const studyStepConfiguration = studyStep?.configuration; + const isAiGraded = Array.isArray(studyStepConfiguration?.services) && studyStepConfiguration.services.some(s => s.type === "nlpRequest"); + const configurationId = getAssessmentConfigurationId(studyStepConfiguration); + const studyName = study?.name || `study_${session?.studyId || "unknown"}`; + + const scoreObject = row.value || {}; + const assessmentState = typeof scoreObject === "string" ? parseAssessmentState(server, scoreObject) : scoreObject; + const flatScores = buildScoresFromState(assessmentState); + const assessmentConfig = resolveAssessmentConfigurationContent( + studyStepConfiguration, + configurationsById + ); + addCriteriaReferenceEntry( + criteriaReferencesByConfigId, + configurationId, + assessmentConfig + ); + const assessmentScore = calculateAssessmentScore(assessmentConfig, flatScores); + const totalPoints = assessmentScore.achieved_points; + + records.push({ + projectId, + userId: ownerUser.id, + userExtId: ownerUser.extId ?? null, + userName: ownerUser.userName ?? "", + displayName: getDisplayName(ownerUser, shouldGenerateAliases, hasPrivateInfoRight, userMapping), + submissionId: submission?.id ?? document.submissionId ?? null, + submissionExtId: submission?.extId ?? null, + studySessionId: row.studySessionId ?? null, + studyStepId: row.studyStepId ?? null, + configurationId, + studyName, + sessionHash: session?.hash ?? null, + studyOwner: getPrivateAwareName(graderUser, hasPrivateInfoRight), + sessionOwner: getPrivateAwareName(reviewerUser, hasPrivateInfoRight), + author: getPrivateAwareName(ownerUser, hasPrivateInfoRight), + scores: flatScores, + totalPoints, + createdAt: row.createdAt ? new Date(row.createdAt).toISOString() : null, + studyStepType: studyStep?.stepType ?? null, + isAiGraded + }); + } + + return { records, criteriaReferencesByConfigId }; +} + +module.exports = { + parseAssessmentState, + getAssessmentConfigurationId, + resolveAssessmentConfigurationContent, + addCriteriaReferenceEntry, + buildGradeCsvRow, + loadGradeExportContext, + compareGradeRecords, + buildGradeRecords, +}; diff --git a/backend/utils/helper/exportProcessors.js b/backend/utils/helper/exportProcessors.js new file mode 100644 index 000000000..723474c27 --- /dev/null +++ b/backend/utils/helper/exportProcessors.js @@ -0,0 +1,616 @@ +const { dbToDelta, deltaToPlainText, deltaToHtml } = require('editor-delta-conversion'); +const { + sanitizeFolderName, + getDisplayName, + getConsentedUserIds, + appendStoredFileIfExists, + appendZipFileAnonymized, + attachTagNames, + createJsonArrayStream, + createCsvRowsStream +} = require('./export.js'); +const { compareGradeRecords, buildGradeRecords } = require('./exportGrades.js'); + +// Stored files the study export ships per step, by document type: PDF and LaTeX ZIP. +const STUDY_DOCUMENT_EXTENSIONS = { 0: '.pdf', 4: '.zip' }; + +/** + * Exports a single document to the archive based on its type. + * - Type 0 (PDF): exports annotations, comments (with votes), document_data, and the PDF file. + * - Type 1 (HTML) / Type 2 (Modal): exports edits, plain text, HTML, and document_data. + * - Type 4 (ZIP): exports the zip file (author name anonymized when aliases are requested) and document_data. + * @param {Object} server - The server instance providing database models. + * @param {Object} doc - The document record from the database. + * @param {string} docFolder - The target folder path inside the archive. + * @param {Object} archive - The archiver instance to append files to. + * @param {boolean} shouldGenerateAliases - Whether ZIP author names should be anonymized. + * @param {Object} userMapping - Map of user IDs to generated aliases. + * @param {Object|null} ownerUser - The document owner's user record (for the real name to replace in ZIPs). + * @returns {Promise} + */ +async function processDocumentForExport(server, doc, docFolder, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, docUserRoles, archive, shouldGenerateAliases, userMapping, ownerUser) { + // document_data for all types, at the doc level. + const documentData = await server.db.models.document_data.findAll({ + where: { documentId: doc.id, deleted: false }, + raw: true, + }); + if (documentData.length > 0) { + archive.append(JSON.stringify(documentData, null, 2), { name: `${docFolder}/document_data.json` }); + } + + const docMeta = { + ...doc.toJSON(), + userRoles: docUserRoles, + }; + archive.append(JSON.stringify(docMeta, null, 2), { name: `${docFolder}/meta.json` }); + + switch (doc.type) { + case 0: { // PDF + // Annotations live on study-session copies (parentDocumentId = doc.id), + // not on the root document. Collect all copy IDs and query across them. + const copies = await server.db.models.document.findAll({ + where: { parentDocumentId: doc.id, deleted: false }, + attributes: ['id'], + raw: true, + }); + const allDocIds = [doc.id, ...copies.map(c => c.id)]; + + let [annotations, comments] = await Promise.all([ + server.db.models.annotation.findAll({ where: { documentId: allDocIds, deleted: false }, raw: true }), + server.db.models.comment.findAll({ where: { documentId: allDocIds, deleted: false }, raw: true }), + ]); + + annotations = await attachTagNames(server, annotations); + + if (shouldExcludeNonConsentingAnnotations) { + const allUserIds = [...new Set([ + ...annotations.map(a => a.userId), + ...comments.map(c => c.userId), + ].filter(Boolean))]; + const consentedUserIds = await getConsentedUserIds(server, allUserIds); + annotations = annotations.filter(a => !a.userId || consentedUserIds.has(a.userId)); + comments = comments.filter(c => !c.userId || consentedUserIds.has(c.userId)); + } + + const commentVotes = await server.db.models.comment_vote.findAll({ + where: { commentId: comments.map(c => c.id), deleted: false }, + raw: true, + }); + const commentsWithVotes = comments.map(c => ({ + ...c, + votes: commentVotes.filter(v => v.commentId === c.id), + })); + + // All annotations and comments go into one file each. + if (annotations.length > 0) { + archive.append(JSON.stringify(annotations, null, 2), { name: `${docFolder}/annotations.json` }); + } + if (commentsWithVotes.length > 0) { + archive.append(JSON.stringify(commentsWithVotes, null, 2), { name: `${docFolder}/comments.json` }); + } + + appendStoredFileIfExists(server, archive, doc.hash, '.pdf', `${docFolder}/document.pdf`, 'PDF'); + break; + } + + case 1: // HTML + case 2: { // MODAL + // fetch all edits for this document, ordered chronologically + let allEdits = await server.db.models.document_edit.findAll({ + where: { documentId: doc.id, deleted: false }, + order: [['createdAt', 'ASC']], + raw: true, + }); + + // filter by consent unless the option is enabled + if (shouldExcludeNonConsentingEdits) { + const editorUserIds = [...new Set(allEdits.map(e => e.userId).filter(Boolean))]; + const consentedUserIds = await getConsentedUserIds(server, editorUserIds); + allEdits = allEdits.filter(e => !e.userId || consentedUserIds.has(e.userId)); + } + + // group edits by studySessionId (null = template) + const sessionGroups = new Map(); + for (const edit of allEdits) { + const key = edit.studySessionId ?? '__template__'; + if (!sessionGroups.has(key)) sessionGroups.set(key, []); + sessionGroups.get(key).push(edit); + } + + // fetch study sessions to resolve hashes + const sessionIds = [...sessionGroups.keys()].filter(k => k !== '__template__'); + const sessions = sessionIds.length > 0 + ? await server.db.models.study_session.findAll({ + where: { id: sessionIds }, + attributes: ['id', 'hash'], + raw: true, + }) + : []; + const sessionHashMap = new Map(sessions.map(s => [s.id, s.hash])); + + for (const [key, edits] of sessionGroups.entries()) { + const isTemplate = key === '__template__'; + const delta = dbToDelta(edits); + + // skip empty content + const text = deltaToPlainText(delta); + if (!text.trim()) continue; + + const subFolder = isTemplate + ? `${docFolder}/template` + : `${docFolder}/${sessionHashMap.get(key) ?? key}`; + + archive.append(text, { name: `${subFolder}/text.txt` }); + archive.append(deltaToHtml(delta), { name: `${subFolder}/html.html` }); + archive.append(JSON.stringify(edits, null, 2), { name: `${subFolder}/edits.json` }); + } + break; + } + + case 4: { // ZIP + await appendZipFileAnonymized(server, archive, doc.hash, `${docFolder}/document.zip`, shouldGenerateAliases, userMapping, ownerUser); + break; + } + + default: + server.logger.warn(`[DocumentExport] Unhandled document type ${doc.type} for document ${doc.hash}, skipping.`); + } +} + +/** + * Main export function for the "documents" export type. + * Fetches all studies and steps for a project, collects unique documents, + * filters by owner data sharing consent, and exports each document to the archive. + * @param {Object} server - The server instance providing database models. + * @param {number|string} projectId - The ID of the project to export. + * @param {Array} userIds - List of user IDs to filter documents by. + * @param {Array} users - Full user records for the selected users (for ZIP anonymization). + * @param {Array} documentTypes - List of document types to include (0=PDF, 1=HTML, 2=Modal, 4=ZIP). + * @param {boolean} shouldGenerateAliases - Whether ZIP author names should be anonymized. + * @param {Object} userMapping - Map of user IDs to generated aliases. + * @param {string} baseFolderName - The root folder name inside the ZIP archive. + * @param {Object} archive - The archiver instance to append files to. + * @returns {Promise} + */ +async function processDocumentBasedExport(server, projectId, userIds, users, documentTypes, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, shouldGenerateAliases, userMapping, baseFolderName, archive) { + try { + documentTypes = typeof documentTypes === 'string' ? JSON.parse(documentTypes) : documentTypes; + if (!Array.isArray(documentTypes)) documentTypes = [0, 1, 2, 4]; + } catch (e) { + server.logger.warn("Could not parse documentTypes:", documentTypes); + documentTypes = [0, 1, 2, 4]; + } + + const docs = await server.db.models.document.findAll({ + where: { projectId, userId: userIds, deleted: false, parentDocumentId: null }, + }); + + if (docs.length === 0) { + server.logger.warn(`[DocumentExport] No documents found for project ${projectId}`); + return; + } + + const filteredDocs = docs.filter(doc => + documentTypes.includes(doc.type) || documentTypes.includes(String(doc.type)) + ); + + if (filteredDocs.length === 0) { + server.logger.warn(`[DocumentExport] No documents matching selected types found for project ${projectId}`); + return; + } + + const uniqueUserIds = [...new Set(filteredDocs.map(doc => doc.userId).filter(Boolean))]; + + const userRoleRows = await server.db.models.user_role_matching.findAll({ + where: { userId: uniqueUserIds }, + raw: true, + }); + + const rolesMap = {}; + for (const row of userRoleRows) { + if (!rolesMap[row.userId]) rolesMap[row.userId] = []; + rolesMap[row.userId].push(row.userRoleId); + } + + const usersById = new Map(users.map(u => [u.id, u])); + + for (const doc of filteredDocs) { + const docFolder = `${baseFolderName}/${doc.hash}`; + const docUserRoles = rolesMap[doc.userId] || []; + await processDocumentForExport(server, doc, docFolder, shouldExcludeNonConsentingEdits, shouldExcludeNonConsentingAnnotations, docUserRoles, archive, shouldGenerateAliases, userMapping, usersById.get(doc.userId) ?? null); + } +} + +/** + * Main export function for the "studies" export type. + * Fetches all studies for the selected users/workflows and, per study session and step, + * archives annotations, comments, edits, document_data, grades, and (optionally) the + * underlying document files. + * @param {Object} server - The server instance providing database models and Sequelize operators. + * @param {number|string} projectId - The ID of the project to export. + * @param {Array} userIds - List of user IDs to filter studies by. + * @param {Array} users - Full user records for the selected users. + * @param {boolean} hasPrivateInfoRight - Whether the requester may export real names. + * @param {Object} userMapping - Map of user IDs to generated aliases. + * @param {Array} workflowIds - Workflow IDs to filter studies by. + * @param {string} baseFolderName - The root folder name inside the ZIP archive. + * @param {Object} archive - The archiver instance to append files to. + * @param {Object} options - Export flags. + * @param {boolean} options.shouldGenerateAliases - Whether student names should be anonymized. + * @param {boolean} options.shouldIncludeEmptyStudies - Whether to include studies/sessions with no exportable content. + * @param {boolean} options.shouldExcludeNonConsentingEdits - Whether to drop edits from users who didn't consent to data sharing. + * @param {boolean} options.shouldExcludeNonConsentingAnnotations - Whether to drop annotations/comments from users who didn't consent to data sharing. + * @param {boolean} options.shouldIncludeDocumentFiles - Whether to include the underlying PDF/ZIP document files per step. + * @param {boolean} options.shouldIncludeGrades - Whether to include grade/score files per session. + * @param {boolean} options.shouldIncludeAiScores - Whether to include AI-assisted scores alongside human grades. + * @returns {Promise} + */ +async function processStudyBasedExport(server, projectId, userIds, users, hasPrivateInfoRight, userMapping, workflowIds, baseFolderName, archive, options) { + const { + shouldGenerateAliases, + shouldIncludeEmptyStudies, + shouldExcludeNonConsentingEdits, + shouldExcludeNonConsentingAnnotations, + shouldIncludeDocumentFiles, + shouldIncludeGrades, + shouldIncludeAiScores, + } = options; + + const usersById = new Map(users.map(u => [u.id, u])); + // A step's document (and its submission siblings) can recur across many sessions in a + // review workflow; cache anonymized ZIP buffers by hash so that work is only done once. + const anonymizedZipBufferCache = new Map(); + + const studyWhere = { userId: userIds, projectId, deleted: false, workflowId: workflowIds }; + + const studies = await server.db.models.study.findAll({ where: studyWhere }); + + if (studies.length === 0) { + server.logger.warn(`[StudyExport] No studies found for selected users in project ${projectId}`); + return; + } + + const writtenCriteriaReferenceIds = new Set(); + + for (const study of studies) { + const studyFolder = `${baseFolderName}/${study.hash}`; + + const sortedSteps = await server.db.models.study_step.getSortedStudySteps(study.id); + + const stepDocumentsById = new Map(); + // A step references one document, but PDF and LaTeX ZIP are separate documents + // of the same submission, each with its own hash. + const submissionDocumentsBySubmissionId = new Map(); + if (shouldIncludeDocumentFiles) { + const stepDocumentIds = [...new Set(sortedSteps.map(step => step.documentId).filter(Boolean))]; + const stepDocuments = stepDocumentIds.length > 0 + ? await server.db.models.document.findAll({ where: { id: stepDocumentIds, deleted: false }, raw: true }) + : []; + for (const doc of stepDocuments) stepDocumentsById.set(doc.id, doc); + + const submissionIds = [...new Set(stepDocuments.map(doc => doc.submissionId).filter(Boolean))]; + const submissionDocuments = submissionIds.length > 0 + ? await server.db.models.document.findAll({ + where: { submissionId: submissionIds, deleted: false }, + raw: true, + }) + : []; + for (const doc of submissionDocuments) { + if (!submissionDocumentsBySubmissionId.has(doc.submissionId)) { + submissionDocumentsBySubmissionId.set(doc.submissionId, []); + } + submissionDocumentsBySubmissionId.get(doc.submissionId).push(doc); + } + } + + const sessions = await server.db.models.study_session.findAll({ + where: { studyId: study.id, deleted: false }, + raw: true, + }); + + const sessionResults = []; + for (const session of sessions) { + const stepResults = []; + let sessionHasContent = false; + + for (let i = 0; i < sortedSteps.length; i++) { + const step = sortedSteps[i]; + const files = []; + + switch (step.stepType) { + case 1: { // Annotator + let annotations = await server.db.models.annotation.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + raw: true, + }); + + annotations = await attachTagNames(server, annotations); + + let comments = await server.db.models.comment.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + raw: true, + }); + + if (shouldExcludeNonConsentingAnnotations) { + const allUserIds = [...new Set([ + ...annotations.map(a => a.userId), + ...comments.map(c => c.userId) + ].filter(Boolean))]; + const consentedIds = await getConsentedUserIds(server, allUserIds); + annotations = annotations.filter(a => !a.userId || consentedIds.has(a.userId)); + comments = comments.filter(c => !c.userId || consentedIds.has(c.userId)); + } + + if (annotations.length > 0) { + files.push({ name: 'annotations.json', content: JSON.stringify(annotations, null, 2) }); + sessionHasContent = true; + } + + if (comments.length > 0) { + const commentVotes = await server.db.models.comment_vote.findAll({ + where: { commentId: comments.map(c => c.id), deleted: false }, + raw: true, + }); + files.push({ + name: 'comments.json', + content: JSON.stringify( + comments.map(c => ({ ...c, votes: commentVotes.filter(v => v.commentId === c.id) })), + null, 2 + ) + }); + sessionHasContent = true; + } + + // Assessments live on the annotator step, so without this the whole + // document_data of an assessment workflow never leaves the database. + const annotatorData = await server.db.models.document_data.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + raw: true, + }); + if (annotatorData.length > 0) { + files.push({ name: 'document_data.json', content: JSON.stringify(annotatorData, null, 2) }); + sessionHasContent = true; + } + break; + } + + case 2: // Editor + case 3: { // Modal + const [templateEdits, sessionEdits] = await Promise.all([ + server.db.models.document_edit.findAll({ + where: { documentId: step.documentId, studySessionId: null, studyStepId: null, deleted: false }, + order: [['createdAt', 'ASC']], + raw: true, + }), + server.db.models.document_edit.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + order: [['createdAt', 'ASC']], + raw: true, + }), + ]); + + let edits = [...templateEdits, ...sessionEdits].sort((a, b) => new Date(a.createdAt) - new Date(b.createdAt)); + + if (shouldExcludeNonConsentingEdits) { + const editorUserIds = [...new Set(edits.map(e => e.userId).filter(Boolean))]; + const consentedIds = await getConsentedUserIds(server, editorUserIds); + edits = edits.filter(e => !e.userId || consentedIds.has(e.userId)); + } + + if (edits.length > 0) { + const delta = dbToDelta(edits); + const text = deltaToPlainText(delta); + if (text.trim()) { + files.push({ name: 'edits.json', content: JSON.stringify(edits, null, 2) }); + files.push({ name: 'text.txt', content: text }); + files.push({ name: 'html.html', content: deltaToHtml(delta) }); + sessionHasContent = true; + } + } + + const documentData = await server.db.models.document_data.findAll({ + where: { documentId: step.documentId, studySessionId: session.id, studyStepId: step.id, deleted: false }, + raw: true, + }); + if (documentData.length > 0) { + files.push({ name: 'document_data.json', content: JSON.stringify(documentData, null, 2) }); + } + break; + } + } + + stepResults.push({ stepIndex: i, files }); + + } + + sessionResults.push({ session, stepResults, hasContent: sessionHasContent }); + } + + const includedSessions = shouldIncludeEmptyStudies + ? sessionResults + : sessionResults.filter(s => s.hasContent); + + if (!shouldIncludeEmptyStudies && includedSessions.length === 0) continue; + + const studyMeta = { + id: study.id, + name: study.name, + userId: study.userId, + workflowId: study.workflowId, + sessions: includedSessions.map(({ session }) => ({ + hash: session.hash, + id: session.id, + userId: session.userId, + numberSteps: session.numberSteps, + steps: sortedSteps.map((step, i) => ({ + id: step.id, + stepNumber: i + 1, + stepType: step.stepType, + configuration: step.configuration, + })) + })) + }; + + archive.append(JSON.stringify(studyMeta, null, 2), { name: `${studyFolder}/meta.json` }); + + let gradeRecordsBySessionId = new Map(); + if (shouldIncludeGrades) { + // Scope by this study's sessions, not by the study owner: review assessments are + // stored on a document owned by the reviewed author, so an owner-scoped lookup + // finds none of them. See buildGradeRecords(). + const { records: ownerGradeRecords, criteriaReferencesByConfigId } = await buildGradeRecords( + server, projectId, [study.userId], users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, + { sessionIds: sessions.map(s => s.id) } + ); + + for (const [configurationId, reference] of criteriaReferencesByConfigId.entries()) { + if (writtenCriteriaReferenceIds.has(configurationId)) continue; + writtenCriteriaReferenceIds.add(configurationId); + archive.append( + JSON.stringify(reference, null, 2), + { name: `${baseFolderName}/criteria_reference_${configurationId}.json` } + ); + } + + for (const record of ownerGradeRecords) { + if (!gradeRecordsBySessionId.has(record.studySessionId)) gradeRecordsBySessionId.set(record.studySessionId, []); + gradeRecordsBySessionId.get(record.studySessionId).push(record); + } + } + + for (const { session, stepResults } of includedSessions) { + const sessionFolder = `${studyFolder}/${session.hash}`; + + if (shouldIncludeGrades) { + const allSessionGrades = (gradeRecordsBySessionId.get(session.id) || []).sort(compareGradeRecords); + + const humanGrades = allSessionGrades.filter(r => !r.isAiGraded).map(({ sessionHash, isAiGraded, ...rest }) => rest); + const aiGrades = allSessionGrades.filter(r => r.isAiGraded).map(({ sessionHash, isAiGraded, ...rest }) => rest); + + if (humanGrades.length > 0) { + archive.append(JSON.stringify(humanGrades, null, 2), { name: `${sessionFolder}/scores.json` }); + } + if (shouldIncludeAiScores && aiGrades.length > 0) { + archive.append(JSON.stringify(aiGrades, null, 2), { name: `${sessionFolder}/scores_ai.json` }); + } + } + + for (const { stepIndex, files } of stepResults) { + const stepFolder = `${sessionFolder}/step_${stepIndex + 1}`; + + if (shouldIncludeDocumentFiles) { + const step = sortedSteps[stepIndex]; + const stepDocument = stepDocumentsById.get(step.documentId); + if (stepDocument) { + const submissionSiblings = stepDocument.submissionId + ? (submissionDocumentsBySubmissionId.get(stepDocument.submissionId) || []) + : []; + const candidates = [ + stepDocument, + ...submissionSiblings.filter(doc => doc.id !== stepDocument.id), + ]; + + const appendedExtensions = new Set(); + for (const doc of candidates) { + const extension = STUDY_DOCUMENT_EXTENSIONS[doc.type]; + if (!extension || appendedExtensions.has(extension)) continue; + appendedExtensions.add(extension); + if (extension === '.zip') { + await appendZipFileAnonymized( + server, + archive, + doc.hash, + `${stepFolder}/document${extension}`, + shouldGenerateAliases, + userMapping, + usersById.get(doc.userId) ?? null, + anonymizedZipBufferCache, + ); + } else { + appendStoredFileIfExists( + server, + archive, + doc.hash, + extension, + `${stepFolder}/document${extension}`, + extension.slice(1).toUpperCase(), + ); + } + } + } + } + + for (const file of files) { + archive.append(file.content, { name: `${stepFolder}/${file.name}` }); + } + } + } + } +} + +/** + * Exports usage statistics for the selected users, respecting each user's acceptStats consent. + * @param {string} behaviourOutputFormat - 'single' for one combined file, 'perUser' for one file per user. + */ +async function processUserBehaviourExport(server, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping, behaviourOutputFormat, behaviourFileFormat, baseFolderName, archive) { + const { Op } = server.db.Sequelize; + + const consentedUsers = users.filter(u => u.acceptStats); + if (consentedUsers.length === 0) return; + const usersById = new Map(consentedUsers.map(u => [u.id, u])); + const consentedUserIds = consentedUsers.map(u => u.id); + const extension = behaviourFileFormat === 'csv' ? 'csv' : 'json'; + + const parseStatData = (raw) => { + try { + return JSON.parse(raw); + } catch (e) { + return raw; + } + }; + + const toRecord = (stat) => { + const user = usersById.get(stat.userId); + return { + action: stat.action, + data: behaviourFileFormat === 'csv' ? stat.data : parseStatData(stat.data), + timestamp: stat.timestamp instanceof Date ? stat.timestamp.toISOString() : stat.timestamp, + user: getDisplayName(user, shouldGenerateAliases, hasPrivateInfoRight, userMapping), + username: user?.userName ?? null, + userId: stat.userId, + session: stat.session, + }; + }; + + const buildStream = (fetchPage) => behaviourFileFormat === 'csv' + ? createCsvRowsStream(fetchPage, toRecord, ['action', 'data', 'timestamp', 'user', 'username', 'userId', 'session']) + : createJsonArrayStream(fetchPage, toRecord); + + if (behaviourOutputFormat === 'perUser') { + for (const user of consentedUsers) { + const folderName = sanitizeFolderName(getDisplayName(user, shouldGenerateAliases, hasPrivateInfoRight, userMapping)); + const fetchPage = (lastId, limit) => server.db.models.statistic.findAll({ + where: { userId: user.id, deleted: false, id: { [Op.gt]: lastId } }, + order: [['id', 'ASC']], + limit, + raw: true, + }); + archive.append(buildStream(fetchPage), { name: `${baseFolderName}/${folderName}/behaviour_data.${extension}` }); + } + } else { + const fetchPage = (lastId, limit) => server.db.models.statistic.findAll({ + where: { userId: { [Op.in]: consentedUserIds }, deleted: false, id: { [Op.gt]: lastId } }, + order: [['id', 'ASC']], + limit, + raw: true, + }); + archive.append(buildStream(fetchPage), { name: `${baseFolderName}/behaviour_data.${extension}` }); + } +} + +module.exports = { + processDocumentForExport, + processDocumentBasedExport, + processStudyBasedExport, + processUserBehaviourExport, +}; diff --git a/backend/webserver/routes/export.js b/backend/webserver/routes/export.js index 35dd32376..9d0a853aa 100644 --- a/backend/webserver/routes/export.js +++ b/backend/webserver/routes/export.js @@ -1,13 +1,29 @@ const archiver = require('archiver'); const path = require('path'); const fs = require('fs'); -const { faker } = require('@faker-js/faker'); -const JSZip = require('jszip'); -const { deriveUserSeed } = require('../auth/utils'); const Papa = require('papaparse'); -const { calculateAssessmentScore, buildScoresFromState } = require('assessment-score'); - -const ASSESSMENT_RESULT_KEY = "assessment_result"; +const { + replaceAuthorInZip, + buildUserMapping, + sanitizeFolderName, + getDisplayName, + calculateSubmissionVersion, + resolveHasPrivateInfoRight, + parseUserIds, + loadExportRequestContext, + resolveIsAdmin, +} = require('../../utils/helper/export.js'); +const { + buildGradeCsvRow, + compareGradeRecords, + buildGradeRecords, +} = require('../../utils/helper/exportGrades.js'); +const { + processDocumentBasedExport, + processStudyBasedExport, + processUserBehaviourExport, +} = require('../../utils/helper/exportProcessors.js'); +const storageDir = path.join(__dirname, "..", "..", "..", "files"); module.exports = function (server) { @@ -16,81 +32,46 @@ module.exports = function (server) { // Auth checking const currentUserId = req.user?.id; if (!currentUserId) return res.status(401).send("Log in required"); - const currentUser = await server.db.models.user.findByPk(currentUserId); if (!currentUser) return res.status(401).send("User not found"); + const hasPrivateInfoRight = await resolveHasPrivateInfoRight(server, currentUserId); - // check if user has right to see full names - let hasPrivateInfoRight = false; - - const roleIds = await server.db.models["user_role_matching"].getUserRolesById(currentUserId); - const isAdmin = await server.db.models["user_role_matching"].isAdminInUserRoles(roleIds); - if (isAdmin) { - // override, admin has all rights - hasPrivateInfoRight = true; - } else { - const userRightsObj = await server.db.models.user.getUserRights(currentUserId); - - if (userRightsObj) { - const allRights = Object.values(userRightsObj).flat(); - hasPrivateInfoRight = allRights.includes('frontend.dashboard.studies.view.userPrivateInfo'); - } - } // Input parsing - const { projectId, exportType, generateAliases, fakerSeed, gradeFormat, mergeCsvFiles } = req.body; - let { userIds = [] } = req.body; + const { projectId, exportType, generateAliases, fakerSeed, gradeFormat, mergeCsvFiles, excludeNonConsentingEdits, excludeNonConsentingAnnotations, includeEmptyStudies, includeDocumentFiles, includeGrades, includeAiScores, behaviourOutputFormat, behaviourFileFormat } = req.body; + let { userIds: rawUserIds = [], documentTypes = [0, 1, 2, 4], workflowIds = [] } = req.body; const shouldGenerateAliases = String(generateAliases) === 'true'; const shouldMergeCsvFiles = String(mergeCsvFiles) === "true"; + const shouldExcludeNonConsentingEdits = String(excludeNonConsentingEdits) === 'true'; + const shouldExcludeNonConsentingAnnotations = String(excludeNonConsentingAnnotations) === 'true'; + const shouldIncludeEmptyStudies = String(includeEmptyStudies) === 'true'; + const shouldIncludeDocumentFiles = String(includeDocumentFiles) === 'true'; + const shouldIncludeGrades = String(includeGrades) === 'true'; + const shouldIncludeAiScores = includeAiScores === undefined ? true : String(includeAiScores) === 'true'; + const normalizedBehaviourOutputFormat = behaviourOutputFormat === 'perUser' ? 'perUser' : 'single'; + const normalizedBehaviourFileFormat = behaviourFileFormat === 'csv' ? 'csv' : 'json'; const normalizedGradeFormat = String(gradeFormat || "json").toLowerCase(); - const supportedExportTypes = new Set(["submissions", "grades"]); - const { Op } = server.db.Sequelize; const parsedProjectId = Number(projectId); + const userIds = parseUserIds(server, rawUserIds); try { - userIds = typeof userIds === 'string' ? JSON.parse(userIds) : userIds; - if (!Array.isArray(userIds)) userIds = []; - } catch (e) { - console.warn("Could not parse userIds:", userIds); - userIds = []; - } - - try { - if (!Number.isInteger(parsedProjectId)) return res.status(400).send("Missing projectId."); - if (!supportedExportTypes.has(exportType)) { - return res.status(400).send("Unsupported export type."); - } - if (exportType === "grades" && !["json", "csv"].includes(normalizedGradeFormat)) { - return res.status(400).send("Unsupported grade format. Use json or csv."); - } - if (userIds.length === 0) { - console.warn("Export aborted: No valid users selected."); - return res.status(400).send("No valid users selected."); - } - - // check if the project is valid - const projectCheck = await server.db.models.project.findOne({ where: { id: parsedProjectId } }); - if (!projectCheck) { - console.warn(`${parsedProjectId} does not exist.`); - return res.status(403).send("The selected project does not exist."); - } - - const users = await server.db.models.user.findAll({ where: { id: { [Op.in]: userIds } } }); - if (users.length === 0) { - console.warn("Export aborted: No existing users to export."); - return res.status(400).send("No authorized users to export."); + const context = await loadExportRequestContext(server, { parsedProjectId, exportType, normalizedGradeFormat, userIds, workflowIds, currentUserId }); + if (!context.success) { + return res.status(context.status).send(context.message); } + const { users, workflowIds: parsedWorkflowIds } = context; // build user mapping for aliases const { userMapping, mappingCsv } = buildUserMapping(users, shouldGenerateAliases, hasPrivateInfoRight, fakerSeed, currentUser.salt); // archiver stream setup - const exportFolderName = `${exportType}_${Date.now()}.zip`; + const timestamp = new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14); + const exportFolderName = `${timestamp}_${exportType}.zip`; res.attachment(exportFolderName); const archive = archiver('zip', { zlib: { level: 5 } }); archive.on('error', function(err) { - console.error("Archiver Error:", err); - if (!res.headersSent) res.status(500).send({error: err.message}); + server.logger.error("Archiver Error:", err); + if (!res.headersSent) res.status(500).send({error: "An error occurred while preparing the export."}); }); // start stream & start by piping the mapping if necessary @@ -99,6 +80,8 @@ module.exports = function (server) { archive.append(mappingCsv, { name: 'aliases_mapping.csv' }); } + const baseFolderName = exportFolderName.split('.')[0]; + // process based on type switch (exportType) { case 'submissions': @@ -110,7 +93,7 @@ module.exports = function (server) { shouldGenerateAliases, hasPrivateInfoRight, userMapping, - exportFolderName.split('.')[0], + baseFolderName, archive ); break; @@ -128,6 +111,61 @@ module.exports = function (server) { archive ); break; + case 'documents': + await processDocumentBasedExport( + server, + parsedProjectId, + userIds, + users, + documentTypes, + shouldExcludeNonConsentingEdits, + shouldExcludeNonConsentingAnnotations, + shouldGenerateAliases, + userMapping, + baseFolderName, + archive + ); + break; + case 'studies': + await processStudyBasedExport( + server, + parsedProjectId, + userIds, + users, + hasPrivateInfoRight, + userMapping, + parsedWorkflowIds, + baseFolderName, + archive, + { + shouldGenerateAliases, + shouldIncludeEmptyStudies, + shouldExcludeNonConsentingEdits, + shouldExcludeNonConsentingAnnotations, + shouldIncludeDocumentFiles, + shouldIncludeGrades, + shouldIncludeAiScores, + } + ); + break; + case 'userBehaviour': { + const isAdmin = await resolveIsAdmin(server, currentUserId); + if (!isAdmin) { + return res.status(403).send("Admin rights required for this export."); + } + await processUserBehaviourExport( + server, + users, + shouldGenerateAliases, + hasPrivateInfoRight, + userMapping, + normalizedBehaviourOutputFormat, + normalizedBehaviourFileFormat, + baseFolderName, + archive + ); + break; + } default: return res.status(400).send("Unsupported export type."); } @@ -135,99 +173,12 @@ module.exports = function (server) { await archive.finalize(); } catch (error) { - console.error("Export Error:", error); + server.logger.error("Export Error:", error); if (!res.headersSent) res.status(500).send("Export failed."); else res.end(); } }); - // HELPER FUNCTIONS - - /** - * Opens a zip file, replaces the student's real name with a fake name in all .tex files, - * and returns the modified zip as a Buffer. - * @param {string} filePath - Path to the original zip file on disk - * @param {string} realName - The student's real name to search for - * @param {string} fakeName - The generated fake name to insert - * @returns {Promise} - The newly generated zip file buffer - */ - async function replaceAuthorInZip(filePath, realName, fakeName) { - const fileData = fs.readFileSync(filePath); - const zip = await JSZip.loadAsync(fileData); - const getFirstAndLastNameTokens = (name) => { - const parts = String(name || "").trim().split(/\s+/).filter(Boolean); - if (parts.length === 0) return ["", ""]; - if (parts.length === 1) return [parts[0], ""]; - return [parts[0], parts[parts.length - 1]]; - }; - const [realFirstName, realLastName] = getFirstAndLastNameTokens(realName); - const [fakeFirstName, fakeLastName] = getFirstAndLastNameTokens(fakeName); - - const authorRegex = /\\author\s*\{[^}]*\}/g; - - for (const [relativePath, zipEntry] of Object.entries(zip.files)) { - if (!zipEntry.dir && relativePath.toLowerCase().endsWith('.tex')) { - let text = await zipEntry.async("string"); - text = text.replace(authorRegex, `\\author{${fakeName}}`); - if (realFirstName && fakeFirstName) text = text.replace(realFirstName, fakeFirstName); - if (realLastName && fakeLastName) text = text.replace(realLastName, fakeLastName); - - zip.file(relativePath, text); - } - } - - return await zip.generateAsync({ - type: "nodebuffer", - compression: "DEFLATE" - }); - } - - /** - * Constructs a mapping of user IDs to aliases and generates a - * corresponding CSV string. - * @param {Array} users - Array of user objects from the database. - * @param {boolean} shouldGenerateAliases - Whether the export should use fake names. - * @param {boolean} hasPrivateInfoRight - Whether the current user is allowed to see/export full names. - * @param {number|string} fakerSeed - The base integer seed (from the form input). - * @param {string} salt - The hex-encoded salt string from the user's database record. - * @returns {Object} An object containing: - * - userMapping: An object mapping user IDs to their generated fake names. - * - mappingCsv: A CSV-formatted string containing the mapping (conditionally includes real names). - */ - function buildUserMapping(users, shouldGenerateAliases, hasPrivateInfoRight, fakerSeed, salt) { - let userMapping = {}; - let csvRows = []; - - if (shouldGenerateAliases) { - if (fakerSeed && !isNaN(parseInt(fakerSeed, 10))) { - const derivedFakerSeed = deriveUserSeed(parseInt(fakerSeed, 10), salt); - faker.seed(derivedFakerSeed); - } - - const sortedUsers = [...users].sort((a, b) => Number(a.id) - Number(b.id)); - sortedUsers.forEach(u => { - const realUsername = u.userName; - const realName = `${u.firstName} ${u.lastName}`; - const fakeName = `${faker.person.firstName()} ${faker.person.lastName()}`; - - userMapping[u.id] = fakeName; - - let rowData = { - "Username": realUsername - }; - if (hasPrivateInfoRight) { - rowData["Real Name"] = realName; - } - - rowData["Generated Alias"] = fakeName; - - csvRows.push(rowData); - }); - } - const mappingCsv = csvRows.length > 0 ? Papa.unparse(csvRows) : ""; - return { userMapping, mappingCsv }; - } - /** * Does the fetching, filtering, and archiving of student submissions for a specific project. * Handles file renaming based on validation rules and manages directory structures @@ -274,7 +225,6 @@ module.exports = function (server) { 1: ".html", 4: ".zip" }; - const storageDir = path.join(__dirname, "..", "..", "..", "files"); for (const submission of submissions) { const student = usersById.get(submission.userId); @@ -312,246 +262,34 @@ module.exports = function (server) { const newZipBuffer = await replaceAuthorInZip(filePath, realName, fakeName); archive.append(newZipBuffer, { name: destPathInArchive }); } catch (err) { - console.error(`Failed to change names for zip ${doc.hash}:`, err); + server.logger.error(`Failed to change names for zip ${doc.hash}:`, err); archive.file(filePath, { name: destPathInArchive }); } } else { archive.file(filePath, { name: destPathInArchive }); } } else { - console.error(`[NOT FOUND] Looking for document: ${doc.hash} at ${filePath}`); + server.logger.error(`[NOT FOUND] Looking for document: ${doc.hash} at ${filePath}`); } } } } /** - * Normalizes a folder name so it can be used as a ZIP path segment without - * accidentally introducing invalid filename characters or nested paths. - * - * @param {string|number|null|undefined} value - The raw folder name. - * @returns {string} A sanitized folder name with reserved characters replaced. + * Serializes grade CSV rows to text, deriving the header from the union of keys across + * every row rather than just the first one's — a session's records can span more than one + * assessment configuration, each contributing its own criterion columns. + * @param {Array} csvRows - Flat rows built by buildGradeCsvRow. + * @returns {string} The CSV text. */ - function sanitizeFolderName(value) { - return String(value || "unknown") - .replace(/[<>:"/\\|?*\x00-\x1F]/g, "_") - .replace(/\s+/g, " ") - .trim(); - } - - /** - * Parses an assessment state payload when it is stored as JSON text. - * - * @param {string} rawAssessmentState - The raw JSON string from document_data. - * @returns {Object} The parsed assessment state or an empty object on failure. - */ - function parseAssessmentState(rawAssessmentState) { - try { - const parsed = JSON.parse(rawAssessmentState); - return parsed && typeof parsed === "object" ? parsed : {}; - } catch (error) { - console.warn("Failed to parse assessment state:", error.message); - return {}; - } - } - - /** - * Resolves the assessment rubric configuration referenced by a study step. - * Study steps are expected to store only a configurationId; rubric content - * is loaded from the configuration table. - * - * @param {Object|null|undefined} studyStepConfiguration - The study step's configuration JSON. - * @param {Map} configurationsById - Loaded configuration records by id. - * @returns {Object|null} Assessment config content (with rubrics) or null. - */ - function resolveAssessmentConfigurationContent(studyStepConfiguration, configurationsById) { - const configurationId = getAssessmentConfigurationId(studyStepConfiguration); - if (configurationId === null) return null; - - const configuration = configurationsById.get(configurationId); - return configuration?.content ?? null; - } - - /** - * Reads the rubric configuration id from a study step configuration payload. - * - * @param {Object|null|undefined} studyStepConfiguration - The study step's configuration object. - * @returns {number|null} The referenced configuration id or null when unavailable. - */ - function getAssessmentConfigurationId(studyStepConfiguration) { - if (!studyStepConfiguration || typeof studyStepConfiguration !== "object") return null; - const rawId = - studyStepConfiguration.settings?.configurationId ?? - studyStepConfiguration.configurationId ?? - null; - const parsedId = Number(rawId); - return Number.isInteger(parsedId) ? parsedId : null; - } - - /** - * Captures the single assessment configuration used by the current grade - * export for inclusion in the shared criteria_reference.json sidecar file. - * - * The first valid configuration becomes the export reference. If another - * different configuration is encountered later, the export aborts because - * grade exports are expected to use exactly one configuration. - * - * @param {{ key: string|null, reference: Object|null }} referenceState - Mutable single-reference state. - * @param {number|null} configurationId - Resolved persisted configuration id. - * @param {Object|null} assessmentConfig - Resolved assessment configuration content. - * @returns {void} - */ - function addCriteriaReferenceEntry(referenceState, configurationId, assessmentConfig) { - if (!assessmentConfig || typeof assessmentConfig !== "object") return; - - const referenceKey = Number.isInteger(configurationId) ? `configuration:${configurationId}` : null; - if (!referenceKey) return; - - if (!referenceState.reference) { - referenceState.key = referenceKey; - referenceState.reference = { - configurationId: Number.isInteger(configurationId) ? configurationId : null, - ...assessmentConfig - }; - return; - } - - if (referenceState.key !== referenceKey) { - throw new Error("Expected exactly one assessment configuration for grade export, found multiple."); + function unparseGradeCsvRows(csvRows) { + const fields = []; + for (const row of csvRows) { + for (const key of Object.keys(row)) { + if (!fields.includes(key)) fields.push(key); + } } - } - - /** - * Returns a user's display name based on private info permissions. - * - * @param {Object|null} user - The user record. - * @param {boolean} hasPrivateInfoRight - Whether real names are allowed. - * @returns {string|null} Full name or username depending on permissions. - */ - function getPrivateAwareName(user, hasPrivateInfoRight) { - if (!user) return null; - if (hasPrivateInfoRight) return `${user.firstName} ${user.lastName}`.trim(); - // Usernames are considered anonymous-enough for exports when real names are restricted. - return user.userName ?? null; - } - - /** - * Builds a flat CSV row for a grade export record. - * The row contains backend export metadata columns followed by - * one column per assessment criterion score. - * - * @param {Object} record - Prepared grade export record. - * @returns {Object} A flat object suitable for Papa.unparse. - */ - function buildGradeCsvRow(record) { - const criterionScores = record.scores && typeof record.scores === "object" ? record.scores : {}; - return { - projectId: record.projectId, - userId: record.userId, - userExtId: record.userExtId, - userName: record.userName, - displayName: record.displayName, - submissionId: record.submissionId, - submissionExtId: record.submissionExtId, - studySessionId: record.studySessionId, - studyName: record.studyName, - studyStepId: record.studyStepId, - studyStepType: record.studyStepType, - configurationId: record.configurationId, - studyOwner: record.studyOwner, - sessionOwner: record.sessionOwner, - author: record.author, - totalPoints: record.totalPoints, - createdAt: record.createdAt, - ...criterionScores - }; - } - - /** - * Resolves the display name for a user based on the current export settings. - * This wraps getPrivateAwareName with alias support for anonymized exports. - * - * @param {Object} user - The user record to display. - * @param {boolean} shouldGenerateAliases - Whether aliases should replace real names. - * @param {boolean} hasPrivateInfoRight - Whether the current user may export real names. - * @param {Object} userMapping - Map of user IDs to generated aliases. - * @returns {string} The display name to write into the export. - */ - function getDisplayName(user, shouldGenerateAliases, hasPrivateInfoRight, userMapping) { - if (shouldGenerateAliases) return userMapping[user.id]; - return getPrivateAwareName(user, hasPrivateInfoRight); - } - - /** - * Loads the related entities needed to turn raw assessment_result rows into - * export-ready grade records. - * - * @param {Object} server - The server instance with Sequelize models. - * @param {Array} gradeRows - Assessment result rows with attached documents. - * @param {Array} users - The selected document owners for the export. - * @returns {Promise} Lookup maps for related grade-export entities. - */ - async function loadGradeExportContext(server, gradeRows, users) { - const { Op } = server.db.Sequelize; - - const sessionIds = [...new Set(gradeRows.map((row) => row.studySessionId).filter(Boolean))]; - const studySessions = sessionIds.length > 0 - ? await server.db.models.study_session.findAll({ - where: { id: { [Op.in]: sessionIds }, deleted: false }, - raw: true - }) - : []; - const sessionsById = new Map(studySessions.map((session) => [session.id, session])); - - const studyIds = [...new Set(studySessions.map((session) => session.studyId).filter(Boolean))]; - const studies = studyIds.length > 0 - ? await server.db.models.study.findAll({ - where: { id: { [Op.in]: studyIds }, deleted: false }, - raw: true - }) - : []; - const studiesById = new Map(studies.map((study) => [study.id, study])); - - const studyStepIds = [...new Set(gradeRows.map((row) => row.studyStepId).filter(Boolean))]; - const studySteps = studyStepIds.length > 0 - ? await server.db.models.study_step.findAll({ - where: { id: { [Op.in]: studyStepIds }, deleted: false }, - raw: true - }) - : []; - const studyStepsById = new Map(studySteps.map((studyStep) => [studyStep.id, studyStep])); - - const configurationIds = [...new Set( - studySteps - .map((studyStep) => getAssessmentConfigurationId(studyStep.configuration)) - .filter((id) => id !== null) - )]; - const configurations = configurationIds.length > 0 - ? await server.db.models.configuration.findAll({ - where: { id: { [Op.in]: configurationIds }, deleted: false }, - raw: true - }) - : []; - const configurationsById = new Map(configurations.map((configuration) => [configuration.id, configuration])); - - // The export references study/session owners in addition to the selected document owners. - const relatedUserIds = [...new Set([ - ...users.map((user) => user.id), - ...studySessions.map((session) => session.userId), - ...studies.map((study) => study.userId) - ].filter(Boolean))]; - const relatedUsers = relatedUserIds.length > 0 - ? await server.db.models.user.findAll({ where: { id: { [Op.in]: relatedUserIds } }, raw: true }) - : []; - const usersById = new Map(relatedUsers.map((user) => [user.id, user])); - - return { - sessionsById, - studiesById, - studyStepsById, - configurationsById, - usersById - }; + return Papa.unparse({ fields, data: csvRows }); } /** @@ -582,116 +320,26 @@ module.exports = function (server) { mergeCsvFiles, archive ) { - const { Op } = server.db.Sequelize; - const gradeRows = await server.db.models.document_data.findAll({ - where: { - key: ASSESSMENT_RESULT_KEY, - deleted: false, - studySessionId: { [Op.ne]: null } - }, - include: [{ - model: server.db.models.document, - as: "document", - // required: true turns this include into an inner join. - required: true, - where: { - projectId, - userId: { [Op.in]: userIds }, - deleted: false - }, - include: [{ - model: server.db.models.submission, - as: "submission", - required: false - }] - }], - // Sort by session first, then step within the session, then creation time within the step. - order: [["studySessionId", "ASC"], ["studyStepId", "ASC"], ["createdAt", "ASC"]] - }); - - const { - sessionsById, - studiesById, - studyStepsById, - configurationsById, - usersById - } = await loadGradeExportContext(server, gradeRows, users); + const { records, criteriaReferencesByConfigId } = await buildGradeRecords( + server, projectId, userIds, users, shouldGenerateAliases, hasPrivateInfoRight, userMapping + ); const recordsByUser = new Map(); - // Grade export currently assumes that all exported rows point to one assessment config. - const criteriaReferenceState = { - key: null, - reference: null - }; - for (const row of gradeRows) { - const document = row.document; - const ownerUser = usersById.get(document.userId); - if (!ownerUser) { - console.warn("Skipping grade export row because the document owner could not be resolved.", { - documentId: document.id, - documentUserId: document.userId, - studySessionId: row.studySessionId, - studyStepId: row.studyStepId - }); - continue; - } - const session = sessionsById.get(row.studySessionId); - const reviewerUser = session ? usersById.get(session.userId) : null; - const study = session ? studiesById.get(session.studyId) : null; - const graderUser = study ? usersById.get(study.userId) : null; - const studyStep = studyStepsById.get(row.studyStepId); - const submission = document.submission; - const studyStepConfiguration = studyStep?.configuration; - // configurationId is exported as metadata; assessmentConfig is the rubric content - // needed for score calculation and criteria_reference.json. - const configurationId = getAssessmentConfigurationId(studyStepConfiguration); - const studyName = study?.name || `study_${session?.studyId || "unknown"}`; - - const scoreObject = row.value || {}; - const assessmentState = typeof scoreObject === "string" ? parseAssessmentState(scoreObject) : scoreObject; - const flatScores = buildScoresFromState(assessmentState); - const assessmentConfig = resolveAssessmentConfigurationContent( - studyStepConfiguration, - configurationsById - ); - addCriteriaReferenceEntry( - criteriaReferenceState, - configurationId, - assessmentConfig - ); - const assessmentScore = calculateAssessmentScore(assessmentConfig, flatScores); - const totalPoints = assessmentScore.achieved_points; - - const record = { - projectId, - userId: ownerUser.id, - userExtId: ownerUser.extId ?? null, - userName: ownerUser.userName ?? "", - displayName: getDisplayName(ownerUser, shouldGenerateAliases, hasPrivateInfoRight, userMapping), - submissionId: submission?.id ?? document.submissionId ?? null, - submissionExtId: submission?.extId ?? null, - studySessionId: row.studySessionId ?? null, - studyStepId: row.studyStepId ?? null, - configurationId, - studyName, - sessionHash: session?.hash ?? null, - studyOwner: getPrivateAwareName(graderUser, hasPrivateInfoRight), - sessionOwner: getPrivateAwareName(reviewerUser, hasPrivateInfoRight), - author: getPrivateAwareName(ownerUser, hasPrivateInfoRight), - scores: flatScores, - totalPoints, - createdAt: row.createdAt ? new Date(row.createdAt).toISOString() : null, - studyStepType: studyStep?.stepType ?? null - }; - - if (!recordsByUser.has(ownerUser.id)) recordsByUser.set(ownerUser.id, []); - recordsByUser.get(ownerUser.id).push(record); + for (const record of records) { + if (!recordsByUser.has(record.userId)) recordsByUser.set(record.userId, []); + recordsByUser.get(record.userId).push(record); } - archive.append( - JSON.stringify(criteriaReferenceState.reference || {}, null, 2), - { name: "grades/criteria_reference.json" } - ); + if (criteriaReferencesByConfigId.size > 0) { + for (const [configurationId, reference] of criteriaReferencesByConfigId.entries()) { + archive.append( + JSON.stringify(reference, null, 2), + { name: `grades/criteria_reference_${configurationId}.json` } + ); + } + } else { + archive.append(JSON.stringify({}, null, 2), { name: "grades/criteria_reference.json" }); + } const usedFolderNames = new Set(); const getUniqueHashFolderName = (baseHash, userId, sessionId) => { @@ -722,38 +370,22 @@ module.exports = function (server) { } for (const [groupKey, groupRecords] of mergedGroups.entries()) { - const sortedRecords = [...groupRecords].sort((a, b) => { - const createdA = a.createdAt ? new Date(a.createdAt).getTime() : 0; - const createdB = b.createdAt ? new Date(b.createdAt).getTime() : 0; - return ( - (a.studySessionId || 0) - (b.studySessionId || 0) || - (a.studyStepId || 0) - (b.studyStepId || 0) || - createdA - createdB - ); - }); + const sortedRecords = [...groupRecords].sort(compareGradeRecords); const csvRows = sortedRecords.map((record) => buildGradeCsvRow(record)); const [studyNamePart, stepIdPart, configurationIdPart] = groupKey.split("__"); const fileName = `${studyNamePart}_${stepIdPart}_${configurationIdPart}.csv`; - archive.append(Papa.unparse(csvRows), { name: `grades/${fileName}` }); + archive.append(unparseGradeCsvRows(csvRows), { name: `grades/${fileName}` }); } return; } for (const user of users) { - const records = (recordsByUser.get(user.id) || []).sort((a, b) => { - const createdA = a.createdAt ? new Date(a.createdAt).getTime() : 0; - const createdB = b.createdAt ? new Date(b.createdAt).getTime() : 0; - return ( - (a.studySessionId || 0) - (b.studySessionId || 0) || - (a.studyStepId || 0) - (b.studyStepId || 0) || - createdA - createdB - ); - }); + const userRecords = (recordsByUser.get(user.id) || []).sort(compareGradeRecords); const recordsByHash = new Map(); - for (const record of records) { + for (const record of userRecords) { const hashKey = record.sessionHash || null; if (!recordsByHash.has(hashKey)) recordsByHash.set(hashKey, []); recordsByHash.get(hashKey).push(record); @@ -766,7 +398,7 @@ module.exports = function (server) { if (gradeFormat === "csv") { const csvRows = exportedRecords.map((record) => buildGradeCsvRow(record)); - archive.append(Papa.unparse(csvRows), { name: `${hashFolder}/scores.csv` }); + archive.append(unparseGradeCsvRows(csvRows), { name: `${hashFolder}/scores.csv` }); } else { archive.append(JSON.stringify(exportedRecords, null, 2), { name: `${hashFolder}/scores.json` }); } @@ -774,23 +406,4 @@ module.exports = function (server) { } } - /** - * Calculates the version number of a submission by traversing backwards - * through the chain of previous submissions. - * @param {Object} submission - The current submission object to start from. - * @param {Map} submissionMap - A Map containing all related - * submissions for quick lookup by ID. - * @returns {number} - The calculated version number (starting at 1 for the original). - */ - function calculateSubmissionVersion(submission, submissionMap) { - let version = 1; - let currentSub = submission; - while (currentSub && currentSub.previousSubmissionId) { - const prevSub = submissionMap.get(currentSub.previousSubmissionId); - if (!prevSub) break; - version++; - currentSub = prevSub; - } - return version; - } -}; +}; \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6ef803a2b..04b2924f1 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -82,7 +82,7 @@ "license": "Apache-2.0", "devDependencies": { "cross-env": "^7.0.3", - "jest": "^30.5.0" + "jest": "^30.5.1" } }, "../utils/modules/editor-delta-conversion": { @@ -90,11 +90,12 @@ "license": "Apache-2.0", "dependencies": { "quill": "2.0.3", - "quill-delta": "^5.1.0" + "quill-delta": "^5.1.0", + "quill-delta-to-html": "0.12.1" }, "devDependencies": { "cross-env": "^7.0.3", - "jest": "^30.5.0" + "jest": "^30.5.1" } }, "node_modules/@babel/code-frame": { @@ -125,6 +126,7 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -1773,6 +1775,7 @@ "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==", "license": "MIT", + "peer": true, "funding": { "type": "opencollective", "url": "https://opencollective.com/popperjs" @@ -2220,6 +2223,7 @@ "integrity": "sha512-oX8xrhvpiyRCQkG1MFchB09f+cXftgIXb3a7UUa4Y3wpmZPw5tyZGTLWhlESOLq1Rq6oDlc8npVU2/9xiCuXMA==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.18.0" } @@ -2587,6 +2591,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2860,6 +2865,7 @@ } ], "license": "MIT", + "peer": true, "peerDependencies": { "@popperjs/core": "^2.11.8" } @@ -2942,6 +2948,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.10.44", "caniuse-lite": "^1.0.30001806", @@ -3969,6 +3976,7 @@ "integrity": "sha512-NPXn6r5zl4uET1DAVPaOwzX3rut4c0wcmw3dWJAfOsTM5+TogXo0DDjz8pwm/hL8cyVNpHqeK4JpN0NjnyFFNw==", "dev": true, "license": "MIT", + "peer": true, "workspaces": [ "packages/*" ], @@ -5459,6 +5467,7 @@ "integrity": "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@keyv/serialize": "^1.1.1" } @@ -6222,6 +6231,7 @@ "resolved": "https://registry.npmjs.org/pg/-/pg-8.23.0.tgz", "integrity": "sha512-Ip2EQCngowJLGOfCwkFhPXU7/ljlhn6Rxlmy4XYfL2Y+vyRM59+8uR2xqRWKdYmbXmxCFOAmKxBuSUCdF34qLg==", "license": "MIT", + "peer": true, "dependencies": { "pg-connection-string": "^2.14.0", "pg-pool": "^3.14.0", @@ -6875,6 +6885,7 @@ "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.59.0.tgz", "integrity": "sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==", "license": "MIT", + "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -7763,6 +7774,7 @@ "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.6.tgz", "integrity": "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==", "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.27.0 || ^0.28.0", "fdir": "^6.5.0", @@ -7902,6 +7914,7 @@ "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.42.tgz", "integrity": "sha512-4RyHQTbQvOPs3MfvUO1Sg0YRrKNnA0mAVtvpd12Tg1fKDN7OHBUl1IqSn8zGJjK9nI3NkNp8cgTpVrSZC5TTcA==", "license": "MIT", + "peer": true, "dependencies": { "@vue/compiler-dom": "3.5.42", "@vue/compiler-sfc": "3.5.42", @@ -7940,6 +7953,7 @@ "integrity": "sha512-Gk6gRDj0n/fkRa3C3l0bBheoBckUq/Rs0F/TvMWIS6nzzx67amAViMe9CkNgsP2tXyQONvGiHQESHwFtZ3aYDA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "debug": "^4.4.0", "eslint-scope": "^8.2.0 || ^9.0.0", diff --git a/frontend/src/components/dashboard/projects/ExportModal.vue b/frontend/src/components/dashboard/projects/ExportModal.vue index e5fdba4e1..09728b7eb 100644 --- a/frontend/src/components/dashboard/projects/ExportModal.vue +++ b/frontend/src/components/dashboard/projects/ExportModal.vue @@ -18,7 +18,9 @@ :fields="dataSelectionFields" /> + - @@ -87,8 +127,11 @@ import JSZip from 'jszip'; import FileSaver from 'file-saver'; import Quill from "quill"; import {dbToDelta} from "editor-delta-conversion"; -import StepSelectStudents from "@/components/dashboard/projects/export/StepSelectStudents.vue"; +import StepSelectUsers from "@/components/dashboard/projects/export/StepSelectUsers.vue"; import StepOptions from "@/components/dashboard/projects/export/StepOptions.vue"; +import StepOptionsDocuments from "@/components/dashboard/projects/export/StepOptionsDocuments.vue"; +import StepOptionsStudies from "@/components/dashboard/projects/export/StepOptionsStudies.vue"; +import StepOptionsUserBehaviour from "@/components/dashboard/projects/export/StepOptionsUserBehaviour.vue"; import StepConfirmDownload from "@/components/dashboard/projects/export/StepConfirmDownload.vue"; import getServerURL from "@/assets/serverUrl.js"; @@ -96,11 +139,11 @@ import getServerURL from "@/assets/serverUrl.js"; /** * ProjectModal - modal component for adding and editing projects * - * @author Dennis Zyska, Mélissa Loew, Linyin Huang + * @author Dennis Zyska, Mélissa Loew */ export default { name: "ExportProjectModal", - components: { StepperModal, BasicForm, StepSelectStudents, StepOptions, StepConfirmDownload }, + components: { StepperModal, BasicForm, StepSelectUsers, StepOptions, StepOptionsDocuments, StepOptionsStudies, StepOptionsUserBehaviour, StepConfirmDownload }, subscribeTable: [{ table: "document", }, { @@ -119,6 +162,18 @@ export default { table: "tag_set", }, { table: "tag" + }, { + table: "document_data", + }, { + table: "study_step", + }, { + table: "configuration", + }, { + table: "workflow", + }, { + table: "user_role", + }, { + table: "user_role_matching", } ], provide() { @@ -135,11 +190,21 @@ export default { filter: [], wait: false, // Data for Export Submissions - submissionSelection: [], + userSelection: [], generateAliases:false, fakerSeed: 846569412, gradeFormat: "json", - mergeCsvFiles: false + mergeCsvFiles: false, + selectedDocumentTypes: [0, 1, 2, 4], + excludeNonConsentingEdits: false, + excludeNonConsentingAnnotations: false, + selectedWorkflowIds: [], + includeStudyDocumentFiles: true, + includeStudyGrades: true, + includeStudyIncludeAiScores: true, + includeEmptyStudies: true, + behaviourOutputFormat: 'single', + behaviourFileFormat: 'json', }; }, computed: { @@ -147,10 +212,31 @@ export default { if (["submissions", "grades"].includes(this.dataSelection.exportType)) { return [ !!this.dataSelection.projectId && !!this.dataSelection.exportType, // must select a valid project and export type - this.submissionSelection.length > 0, // must select at least one student + this.userSelection.length > 0, // must select at least one student true, true ]; + } else if (this.dataSelection.exportType === "documents") { + return [ + !!this.dataSelection.projectId && !!this.dataSelection.exportType, + this.userSelection.length > 0, + this.selectedDocumentTypes.length > 0, + true, + ]; + } else if (this.dataSelection.exportType === 'studies') { + return [ + !!this.dataSelection.projectId && !!this.dataSelection.exportType, + this.userSelection.length > 0, + this.selectedWorkflowIds.length > 0, + true, + ]; + } else if (this.dataSelection.exportType === 'userBehaviour') { + return [ + !!this.dataSelection.projectId && !!this.dataSelection.exportType, + this.userSelection.length > 0, + true, + true, + ]; } return [ !!this.dataSelection.projectId && !!this.dataSelection.exportType, @@ -158,7 +244,7 @@ export default { ]; }, steps() { - if (["submissions", "grades"].includes(this.dataSelection.exportType)) { + if (["submissions", "grades", "documents", "studies", "userBehaviour"].includes(this.dataSelection.exportType)) { return [ { title: this.$t('settings.title') }, { title: this.$t('dashboard.projects.exportModal.steps.selectStudent') }, @@ -192,6 +278,9 @@ export default { {name: this.$t('dashboard.projects.exportModal.exportReviewers'), value: "reviewerList"}, {name: this.$t('dashboard.projects.exportModal.exportSubmissions'), value: "submissions"}, {name: this.$t('dashboard.projects.exportModal.exportGrades'), value: "grades"}, + {name: this.$t('dashboard.projects.exportModal.exportDocuments'), value: "documents"}, + {name: this.$t('dashboard.projects.exportModal.exportStudies'), value: "studies"}, + ...(this.$store.getters["auth/isAdmin"] ? [{name: this.$t('dashboard.projects.exportModal.exportUserBehaviour'), value: "userBehaviour"}] : []), {name: this.$t('common.all'), value: "all"}, ], required: true, @@ -250,13 +339,40 @@ export default { return this.$store.getters["table/project/getAll"]; }, }, + watch: { + 'dataSelection.exportType'() { + this.resetOptions(); + }, + 'dataSelection.projectId'() { + this.resetOptions(); + } + }, methods: { + resetOptions() { + this.filter = []; + this.userSelection = []; + this.generateAliases = false; + this.fakerSeed = 846569412; + this.gradeFormat = "json"; + this.mergeCsvFiles = false; + this.selectedDocumentTypes = [0, 1, 2, 4]; + this.excludeNonConsentingEdits = false; + this.excludeNonConsentingAnnotations = false; + this.selectedWorkflowIds = []; + this.includeStudyDocumentFiles = true; + this.includeStudyGrades = true; + this.includeStudyIncludeAiScores = true; + this.includeEmptyStudies = true; + this.behaviourOutputFormat = 'single'; + this.behaviourFileFormat = 'json'; + }, open(projectId) { this.dataSelection.projectId = projectId; this.$refs.exportStepper.open(); }, hide() { - this.filter = []; + this.resetOptions(); + this.wait = false; }, downloadData() { if (this.dataSelection.exportType === "reviewerList") { @@ -265,6 +381,12 @@ export default { this.downloadSubmissions(); } else if (this.dataSelection.exportType === "grades") { this.downloadGrades(); + } else if (this.dataSelection.exportType === "documents") { + this.downloadDocuments(); + } else if (this.dataSelection.exportType === 'studies') { + this.downloadStudies(); + } else if (this.dataSelection.exportType === 'userBehaviour') { + this.downloadUserBehaviour(); } else { this.downloadAllData(); } @@ -328,7 +450,7 @@ export default { async downloadSubmissions() { try { // get the selected student's user ids - const selectedUserIds = this.submissionSelection.map(row => row.userId); + const selectedUserIds = this.userSelection.map(row => row.userId); // call helper function to trigger the stream download this.triggerStreamDownload({ projectId: this.dataSelection.projectId, @@ -345,7 +467,7 @@ export default { }, async downloadGrades() { try { - const selectedUserIds = this.submissionSelection.map(row => row.userId); + const selectedUserIds = this.userSelection.map(row => row.userId); this.triggerStreamDownload({ projectId: this.dataSelection.projectId, exportType: 'grades', @@ -361,6 +483,65 @@ export default { this.$toast.error("An error occurred starting the stream. Please try again."); } }, + async downloadDocuments() { + try { + const selectedUserIds = this.userSelection.map(row => row.userId); + this.triggerStreamDownload({ + projectId: this.dataSelection.projectId, + exportType: 'documents', + userIds: selectedUserIds, + documentTypes: this.selectedDocumentTypes, + excludeNonConsentingEdits: this.excludeNonConsentingEdits, + excludeNonConsentingAnnotations: this.excludeNonConsentingAnnotations, + generateAliases: this.generateAliases, + fakerSeed: this.generateAliases ? this.fakerSeed : null + }); + + this.$refs.exportStepper.close(); + } catch (error) { + console.error("Streaming error:", error); + this.$toast.error("An error occurred starting the stream. Please try again."); + } + }, + async downloadStudies() { + try { + const selectedUserIds = this.userSelection.map(row => row.userId); + this.triggerStreamDownload({ + projectId: this.dataSelection.projectId, + exportType: 'studies', + userIds: selectedUserIds, + workflowIds: this.selectedWorkflowIds, + includeEmptyStudies: this.includeEmptyStudies, + includeDocumentFiles: this.includeStudyDocumentFiles, + includeGrades: this.includeStudyGrades, + excludeNonConsentingEdits: this.excludeNonConsentingEdits, + excludeNonConsentingAnnotations: this.excludeNonConsentingAnnotations, + includeAiScores: this.includeStudyIncludeAiScores, + generateAliases: this.generateAliases, + fakerSeed: this.generateAliases ? this.fakerSeed : null + }); + this.$refs.exportStepper.close(); + } catch (error) { + console.error("Streaming error:", error); + this.$toast.error("An error occurred starting the stream. Please try again."); + } + }, + async downloadUserBehaviour() { + try { + const selectedUserIds = this.userSelection.map(row => row.userId); + this.triggerStreamDownload({ + projectId: this.dataSelection.projectId, + exportType: 'userBehaviour', + userIds: selectedUserIds, + behaviourOutputFormat: this.behaviourOutputFormat, + behaviourFileFormat: this.behaviourFileFormat, + }); + this.$refs.exportStepper.close(); + } catch (error) { + console.error("Streaming error:", error); + this.$toast.error("An error occurred starting the stream. Please try again."); + } + }, async downloadAllData() { this.wait = true; diff --git a/frontend/src/components/dashboard/projects/export/StepConfirmDownload.vue b/frontend/src/components/dashboard/projects/export/StepConfirmDownload.vue index 1e7e580ce..979e42d5e 100644 --- a/frontend/src/components/dashboard/projects/export/StepConfirmDownload.vue +++ b/frontend/src/components/dashboard/projects/export/StepConfirmDownload.vue @@ -6,14 +6,14 @@
{{ $t('dashboard.projects.export.confirmSelection') }}
- +
@@ -29,16 +29,19 @@
{{ $t('dashboard.projects.export.summary') }}
+
    -
  • - {{ row.studentName || row.userName }} ({{ $t('dashboard.projects.export.fileCount', { count: row.fileCount }) }}) +
  • + {{ row.name }} ({{ row.suffix }})
@@ -54,13 +57,12 @@ import BasicLoading from "@/basic/Loading.vue"; * * The final confirmation step within the ExportModal. * This component provides a summary of the selected - * submissions intended for download, as well as some + * data intended for download, as well as some * warnings for the user, if they selected generate aliases * or students who didn't accept data sharing. * * @author Mélissa Loew */ - export default { name: "StepConfirmDownload", components: { BasicLoading }, @@ -73,15 +75,49 @@ export default { type: Boolean, default: false }, - submissionSelection: { + userSelection: { type: Array, required: true + }, + exportType: { + type: String, + default: 'submissions' } }, computed: { hasDeclinedSharingSelected() { - return this.submissionSelection.some(row => row.acceptDataSharing === false); - } + return this.exportType === 'userBehaviour' + ? this.userSelection.some(row => row.acceptStatsSharing === false) + : this.userSelection.some(row => row.acceptDataSharing === false); + }, + declinedSharingWarningKey() { + return this.exportType === 'userBehaviour' + ? 'dashboard.projects.export.declinedStatsSharingWarning' + : 'dashboard.projects.export.declinedSharingWarning'; + }, + declinedSharingEmphasisKey() { + return this.exportType === 'userBehaviour' + ? 'dashboard.projects.export.declinedStatsSharingEmphasis' + : 'dashboard.projects.export.declinedSharingEmphasis'; + }, + exportTypeLabel() { + const labels = this.$tm('dashboard.projects.export.typeLabel'); + return labels[this.exportType] || labels.documents; + }, + userSelectionDisplay() { + const unitKeyByExportType = { + submissions: 'submissions', + studies: 'studies', + }; + const unitKey = unitKeyByExportType[this.exportType] || 'documents'; + return this.userSelection.map(row => ({ + userId: row.userId, + name: row.fullName || row.userName, + suffix: ['grades', 'userBehaviour'].includes(this.exportType) + ? null + : this.$t(`dashboard.projects.export.unitCount.${unitKey}`, { count: row.count }), + })); + }, } } \ No newline at end of file diff --git a/frontend/src/components/dashboard/projects/export/StepOptionsDocuments.vue b/frontend/src/components/dashboard/projects/export/StepOptionsDocuments.vue new file mode 100644 index 000000000..095d63345 --- /dev/null +++ b/frontend/src/components/dashboard/projects/export/StepOptionsDocuments.vue @@ -0,0 +1,149 @@ + + + \ No newline at end of file diff --git a/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue b/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue new file mode 100644 index 000000000..2d122aece --- /dev/null +++ b/frontend/src/components/dashboard/projects/export/StepOptionsStudies.vue @@ -0,0 +1,251 @@ + + + \ No newline at end of file diff --git a/frontend/src/components/dashboard/projects/export/StepOptionsUserBehaviour.vue b/frontend/src/components/dashboard/projects/export/StepOptionsUserBehaviour.vue new file mode 100644 index 000000000..421955749 --- /dev/null +++ b/frontend/src/components/dashboard/projects/export/StepOptionsUserBehaviour.vue @@ -0,0 +1,68 @@ + + + \ No newline at end of file diff --git a/frontend/src/components/dashboard/projects/export/StepSelectStudents.vue b/frontend/src/components/dashboard/projects/export/StepSelectStudents.vue deleted file mode 100644 index 0653e831f..000000000 --- a/frontend/src/components/dashboard/projects/export/StepSelectStudents.vue +++ /dev/null @@ -1,154 +0,0 @@ - - - \ No newline at end of file diff --git a/frontend/src/components/dashboard/projects/export/StepSelectUsers.vue b/frontend/src/components/dashboard/projects/export/StepSelectUsers.vue new file mode 100644 index 000000000..66255ecbb --- /dev/null +++ b/frontend/src/components/dashboard/projects/export/StepSelectUsers.vue @@ -0,0 +1,329 @@ + + + \ No newline at end of file diff --git a/utils/modules/editor-delta-conversion/index.js b/utils/modules/editor-delta-conversion/index.js index c8d722197..763b6e097 100644 --- a/utils/modules/editor-delta-conversion/index.js +++ b/utils/modules/editor-delta-conversion/index.js @@ -2,10 +2,11 @@ * * This module provides methods to convert between Quill Delta objects and database entries. * - * @author Juliane Bechert + * @author Juliane Bechert, Mélissa Loew * */ const Delta = require('quill-delta'); +const { QuillDeltaToHtmlConverter } = require('quill-delta-to-html'); /** * Converts an array of database entries to a Quill Delta object. @@ -134,8 +135,29 @@ function deltaToPlainText(deltaOrOps) { .join(""); } +/** + * Converts a Quill Delta object to an HTML string. + * Each newline in the delta marks the end of a paragraph and is flushed as a

tag. + * Supports bold, italic, underline, and link attributes. + * + * @param {object|array} deltaOrOps - Quill Delta ({ ops: [...] }) or ops array + * @returns {string} A full HTML document string + */ +function deltaToHtml(deltaOrOps) { + if (!deltaOrOps) return ""; + const ops = Array.isArray(deltaOrOps) + ? deltaOrOps + : (deltaOrOps.ops || []); + + const converter = new QuillDeltaToHtmlConverter(ops, {}); + const body = converter.convert(); + + return `\n\n\n${body}\n`; +} + module.exports = { deltaToDb: deltaToDb, dbToDelta: dbToDelta, deltaToPlainText: deltaToPlainText, + deltaToHtml: deltaToHtml, } \ No newline at end of file diff --git a/utils/modules/editor-delta-conversion/package-lock.json b/utils/modules/editor-delta-conversion/package-lock.json index 4b79017e8..235d87230 100644 --- a/utils/modules/editor-delta-conversion/package-lock.json +++ b/utils/modules/editor-delta-conversion/package-lock.json @@ -10,7 +10,8 @@ "license": "Apache-2.0", "dependencies": { "quill": "2.0.3", - "quill-delta": "^5.1.0" + "quill-delta": "^5.1.0", + "quill-delta-to-html": "0.12.1" }, "devDependencies": { "cross-env": "^7.0.3", @@ -48,6 +49,7 @@ "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.7", "@babel/generator": "^7.29.7", @@ -513,29 +515,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@emnapi/core": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", - "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.1", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", - "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@emnapi/wasi-threads": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", @@ -2116,6 +2095,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.11.20", "caniuse-lite": "^1.0.30001810", @@ -4005,6 +3985,15 @@ "node": ">= 12.0.0" } }, + "node_modules/quill-delta-to-html": { + "version": "0.12.1", + "resolved": "https://registry.npmjs.org/quill-delta-to-html/-/quill-delta-to-html-0.12.1.tgz", + "integrity": "sha512-QhpeMk9+5ge3HYbL5A0Ewz3pXCsbemqGvIF/kw5D6D4V68AtcUp7yt9xNUkzOk/0IQz43hKy3IkzBzRhLIE+oA==", + "license": "ISC", + "dependencies": { + "lodash.isequal": "^4.5.0" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", diff --git a/utils/modules/editor-delta-conversion/package.json b/utils/modules/editor-delta-conversion/package.json index 9283a1619..d3a24f43f 100644 --- a/utils/modules/editor-delta-conversion/package.json +++ b/utils/modules/editor-delta-conversion/package.json @@ -18,7 +18,8 @@ "license": "Apache-2.0", "dependencies": { "quill": "2.0.3", - "quill-delta": "^5.1.0" + "quill-delta": "^5.1.0", + "quill-delta-to-html": "0.12.1" }, "devDependencies": { "cross-env": "^7.0.3", diff --git a/utils/modules/i18n/de/common.json b/utils/modules/i18n/de/common.json index 270960615..3aee2c646 100644 --- a/utils/modules/i18n/de/common.json +++ b/utils/modules/i18n/de/common.json @@ -138,6 +138,8 @@ "description": "Beschreibung", "running": "Läuft", "finished": "Beendet", + "selectAll": "Alle auswählen", + "unselectAll": "Alle abwählen", "nlp": { "serviceRequest": "NLP-Dienstanfrage" } diff --git a/utils/modules/i18n/de/dashboard.json b/utils/modules/i18n/de/dashboard.json index fb0d44492..f5a0dfff5 100644 --- a/utils/modules/i18n/de/dashboard.json +++ b/utils/modules/i18n/de/dashboard.json @@ -51,21 +51,31 @@ "confirmSelection": "Auswahl bestätigen:", "declinedSharingWarning": "Sie haben einen oder mehrere Studierende ausgewählt, die der Datenfreigabe {emphasis}.", "declinedSharingEmphasis": "nicht zugestimmt haben", + "declinedStatsSharingWarning": "Sie haben einen oder mehrere Benutzer ausgewählt, die der Freigabe des Nutzerverhaltens {emphasis}.", + "declinedStatsSharingEmphasis": "nicht zugestimmt haben", "aliasMappingWarning": "Das heruntergeladene ZIP-Archiv enthält eine CSV-Datei, die die generierten Aliasnamen den echten Namen der Studierenden zuordnet.", "reviewWarning": "Wir empfehlen dringend, diesen Export vor der Weitergabe sorgfältig zu prüfen, um sicherzustellen, dass keine sensiblen oder unbeabsichtigten Informationen verteilt werden.", "summary": "Zusammenfassung:", - "downloadSummary": "Sie sind dabei, Abgaben für {count} Studierende herunterzuladen.", - "fileCount": "{count} Dateien" + "downloadSummary": "Sie sind dabei, {type} für {count} Benutzer herunterzuladen.", + "typeLabel": { + "submissions": "Submissions", + "grades": "Noten", + "documents": "Dokumente", + "studies": "Studien", + "userBehaviour": "Daten zum Nutzerverhalten" + }, + "unitCount": { + "submissions": "{count} Submission(s)", + "studies": "{count} Studie(n)", + "documents": "{count} Dokument(e)" + } }, "columns": { "name": "Projektname", "public": "Öffentlich", "closed": "Geschlossen", "username": "Benutzername", - "studentName": "Name des Studierenden", - "files": "Dateien", - "acceptedDataSharing": "Datenfreigabe akzeptiert", - "lastSubmitted": "Zuletzt eingereicht" + "acceptedDataSharing": "Datenfreigabe akzeptiert" }, "actions": { "copy": "Projekt kopieren", @@ -104,10 +114,49 @@ "noteLabel": "Hinweis:", "accountSpecificHint": "Aliase sind an Ihr Konto gebunden und stimmen nicht mit den Exporten anderer Benutzer überein.", "gradeFileFormat": "Dateiformat für Noten", - "mergeCsvFiles": "CSV-Dateien nach Studie, Schritt und Konfiguration zusammenführen" + "mergeCsvFiles": "CSV-Dateien nach Studie, Schritt und Konfiguration zusammenführen", + "userBehaviour": { + "title": "Optionen für Nutzerverhalten", + "fileLayout": "Dateilayout", + "singleCombinedFile": "Eine zusammengeführte Datei", + "onePerUser": "Eine Datei pro Benutzer", + "fileFormat": "Dateiformat" + }, + "excludeNonConsentingEdits": "Bearbeitungen von Benutzern ohne Einwilligung ausschließen", + "excludeNonConsentingAnnotations": "Annotationen und Kommentare von Benutzern ohne Einwilligung ausschließen", + "documents": { + "title": "Dokumentenoptionen", + "typesToInclude": "Einzuschließende Dokumenttypen", + "typePdf": "PDF — Enthält Annotationen und Kommentare", + "typeHtml": "HTML — Enthält Bearbeitungen, Klartext und HTML", + "typeModal": "Modal — Enthält Bearbeitungen, Klartext und HTML", + "typeZip": "ZIP — Enthält die ZIP-Datei" + }, + "studies": { + "title": "Studienoptionen", + "noWorkflowsFound": "Keine Workflows für dieses Projekt gefunden.", + "filterByWorkflow": "Nach Workflow filtern", + "noWorkflowsSelected": "Keine Workflows ausgewählt", + "allWorkflowsSelected": "Alle Workflows ausgewählt", + "includeEmptyStudies": "Studien ohne Sitzungen einschließen", + "includeDocumentFiles": "PDFs und ZIP-Dateien einschließen", + "includeScores": "Bewertungen einschließen", + "includeAiScores": "KI-gestützte Bewertungen einschließen" + } + }, + "exportSelectUsers": { + "title": "Benutzer für den Datenexport auswählen:", + "noUsersFound": "Keine Benutzer mit Daten für dieses Projekt gefunden.", + "fullName": "Vollständiger Name", + "roles": "Rollen", + "acceptBehaviourSharing": "Freigabe des Nutzerverhaltens akzeptiert", + "submissions": "Submissions", + "documents": "Dokumente", + "studies": "Studien", + "assessmentConfigurations": "Bewertungskonfiguration(en)", + "noConfiguration": "Keine Konfiguration", + "noRole": "Keine Rolle" }, - "exportSelectSubmissions": "Einreichungen zum Herunterladen auswählen:", - "noSubmissionsFound": "Keine Einreichungen für dieses Projekt gefunden.", "exportModal": { "steps": { "selectStudent": "Student auswählen", @@ -117,7 +166,10 @@ }, "exportReviewers": "Liste aller Reviewer exportieren", "exportSubmissions": "Submissions exportieren", - "exportGrades": "Noten exportieren" + "exportGrades": "Noten exportieren", + "exportDocuments": "Dokumente exportieren", + "exportStudies": "Studien exportieren", + "exportUserBehaviour": "Nutzerverhalten exportieren" } }, "settings": { diff --git a/utils/modules/i18n/en/common.json b/utils/modules/i18n/en/common.json index fb415d819..6f5ede17a 100644 --- a/utils/modules/i18n/en/common.json +++ b/utils/modules/i18n/en/common.json @@ -138,6 +138,8 @@ "description": "Description", "running": "Running", "finished": "Finished", + "selectAll": "Select All", + "unselectAll": "Unselect All", "nlp": { "serviceRequest": "NLP Service Request" } diff --git a/utils/modules/i18n/en/dashboard.json b/utils/modules/i18n/en/dashboard.json index bcb9a8fd6..0a81874ee 100644 --- a/utils/modules/i18n/en/dashboard.json +++ b/utils/modules/i18n/en/dashboard.json @@ -51,21 +51,31 @@ "confirmSelection": "Confirm Selection:", "declinedSharingWarning": "You have selected one or more students who {emphasis}.", "declinedSharingEmphasis": "didn't accept data sharing", + "declinedStatsSharingWarning": "You have selected one or more users who {emphasis}.", + "declinedStatsSharingEmphasis": "didn't accept behaviour sharing", "aliasMappingWarning": "The downloaded ZIP archive will include a CSV file that maps the generated aliases back to the real student names.", "reviewWarning": "We strongly recommend conducting a thorough review of this export before sharing it, to ensure no sensitive or unintended information is distributed.", "summary": "Summary:", - "downloadSummary": "You are about to download submissions for {count} student(s).", - "fileCount": "{count} files" + "downloadSummary": "You are about to download {type} for {count} user(s).", + "typeLabel": { + "submissions": "submissions", + "grades": "grades", + "documents": "documents", + "studies": "studies", + "userBehaviour": "user behaviour data" + }, + "unitCount": { + "submissions": "{count} submission(s)", + "studies": "{count} study(ies)", + "documents": "{count} document(s)" + } }, "columns": { "name": "Project name", "public": "Public", "closed": "Closed", "username": "Username", - "studentName": "Student Name", - "files": "Files", - "acceptedDataSharing": "Accepted Data Sharing", - "lastSubmitted": "Last Submitted" + "acceptedDataSharing": "Accepted Data Sharing" }, "actions": { "copy": "Copy project", @@ -104,10 +114,49 @@ "noteLabel": "Note:", "accountSpecificHint": "Aliases are tied to your account and won't match other users' exports.", "gradeFileFormat": "Grade file format", - "mergeCsvFiles": "Merge CSV files by study, step, and configuration" + "mergeCsvFiles": "Merge CSV files by study, step, and configuration", + "userBehaviour": { + "title": "User Behaviour Options", + "fileLayout": "File Layout", + "singleCombinedFile": "Single combined file", + "onePerUser": "One file per user", + "fileFormat": "File Format" + }, + "excludeNonConsentingEdits": "Exclude edits from non-consenting users", + "excludeNonConsentingAnnotations": "Exclude annotations and comments from non-consenting users", + "documents": { + "title": "Document Options", + "typesToInclude": "Document Types to Include", + "typePdf": "PDF — Includes annotations and comments", + "typeHtml": "HTML — Includes edits, plain text and HTML", + "typeModal": "Modal — Includes edits, plain text and HTML", + "typeZip": "ZIP — Includes the zip file" + }, + "studies": { + "title": "Study Options", + "noWorkflowsFound": "No workflows found for this project.", + "filterByWorkflow": "Filter by Workflow", + "noWorkflowsSelected": "No workflows selected", + "allWorkflowsSelected": "All workflows selected", + "includeEmptyStudies": "Include studies with no sessions", + "includeDocumentFiles": "Include PDFs and ZIP files", + "includeScores": "Include scores", + "includeAiScores": "Include AI-assisted scores" + } + }, + "exportSelectUsers": { + "title": "Select Users for the Data Export:", + "noUsersFound": "No users with data found for this project.", + "fullName": "Full Name", + "roles": "Roles", + "acceptBehaviourSharing": "Accept Behaviour Sharing", + "submissions": "Submissions", + "documents": "Documents", + "studies": "Studies", + "assessmentConfigurations": "Assessment Configuration(s)", + "noConfiguration": "No configuration", + "noRole": "No role" }, - "exportSelectSubmissions": "Select Submissions to Download:", - "noSubmissionsFound": "No submissions found for this project.", "exportModal": { "steps": { "selectStudent": "Select Student", @@ -117,7 +166,10 @@ }, "exportReviewers": "Export a list of all reviewers", "exportSubmissions": "Export submissions", - "exportGrades": "Export grades" + "exportGrades": "Export grades", + "exportDocuments": "Export documents", + "exportStudies": "Export studies", + "exportUserBehaviour": "Export user behaviour" } }, "settings": {