diff --git a/tools/cldr-apps/js/src/esm/cldrDash.mjs b/tools/cldr-apps/js/src/esm/cldrDash.mjs index cdfde9ed4d6..3509405024a 100644 --- a/tools/cldr-apps/js/src/esm/cldrDash.mjs +++ b/tools/cldr-apps/js/src/esm/cldrDash.mjs @@ -2,38 +2,287 @@ * cldrDash: encapsulate dashboard data. */ import * as cldrAjax from "./cldrAjax.mjs"; +import * as cldrCoverage from "./cldrCoverage.mjs"; import * as cldrNotify from "./cldrNotify.mjs"; import * as cldrProgress from "./cldrProgress.mjs"; import * as cldrStatus from "./cldrStatus.mjs"; import * as cldrSurvey from "./cldrSurvey.mjs"; -import * as cldrXlsx from "./cldrXlsx.mjs"; import * as XLSX from "xlsx"; -/** - * An object whose keys are xpstrid (xpath hex IDs like "db7b4f2df0427e4"), and whose values are objects whose - * keys are notification categories such as "Error" or "English_Changed" and values are "entry" - * objects with code, english, ..., xpstrid, ... elements. - * - * Example: pathIndex[xpstrid][category] = entry - * - * There can be at most one notification for a path within each category, but the same path may have notifications in - * multiple categories. For example, the path with xpstrid = "db7b4f2df0427e4" might have both "Error" and "Warning" notifications, - * but there can't be more than one "Error" notification for the same path, since the server will have combined them - * into a single "Error" notification with multiple error messages. - */ -let pathIndex = {}; +class DashData { + /** + * Construct a new DashData object + * + * @returns the new DashData object + */ + constructor() { + this.entries = []; // array of DashEntry objects + this.cats = new Set(); // set of category names, such as "Error" + this.catSize = {}; // number of entries in each category + this.catFirst = {}; // first entry.xpstrid in each category + // An object whose keys are xpstrid (xpath hex IDs like "db7b4f2df0427e4"), and whose values are DashEntry objects + this.pathIndex = {}; + this.hiddenObject = null; + } + + addEntriesFromJson(notifications) { + for (let catData of notifications) { + for (let group of catData.groups) { + for (let e of group.entries) { + this.addEntry(catData.category, group, e); + } + } + } + } + + /** + * Create a new DashEntry in this DashData, or update the existing entry if already present based on e.xpstrid + * + * @param {String} cat the category such as "Error" + * @param {Object} group (from json) + * @param {Object} e (entry in old format, from json) + */ + addEntry(cat, group, e) { + this.addCategory(cat); + this.catSize[cat]++; + if (!this.catFirst[cat]) { + this.catFirst[cat] = e.xpstrid; + } + if (this.pathIndex[e.xpstrid]) { + const dashEntry = this.pathIndex[e.xpstrid]; + dashEntry.addCategory(cat); + dashEntry.setWinning(e.winning); + if (e.comment) { + dashEntry.setComment(e.comment); + } + if (e.subtype) { + dashEntry.setSubtype(e.subtype); + } + } else { + const dashEntry = new DashEntry(e.xpstrid, e.code, e.english); + dashEntry.setSectionPageHeader(group.section, group.page, group.header); + dashEntry.addCategory(cat); + dashEntry.setWinning(e.winning); + dashEntry.setPreviousEnglish(e.previousEnglish); + dashEntry.setComment(e.comment); + dashEntry.setSubtype(e.subtype); + dashEntry.setChecked(this.itemIsChecked(e)); + this.entries.push(dashEntry); + this.pathIndex[e.xpstrid] = dashEntry; + } + } + + /** + * If this category is not already present in the data, add it and + * initialize its size to zero. No effect if the category is already present. + * + * @param {String} cat + */ + addCategory(cat) { + if (!this.cats.has(cat)) { + this.cats.add(cat); + this.catSize[cat] = 0; + } + } + + /** + * Set the object indicating which paths the user wants hidden in the dashboard + * + * The format matches the json currently returned by the back end. The keys are + * "subtype" strings such as "incorrectCasing"; the values are arrays of objects + * each of which has an xpstrid and a value string + * + * @param {Object} hiddenObject + * Example of json.hidden: + * { + "incorrectCasing": [ + { + "value": "région micronésienne", + "xpstrid": "fe9015c6c61370" + }, + { + "value": "régions éloignées de l’Océanie", + "xpstrid": "744c18884a1547be" + }, + { + "value": "pseudo-accents", + "xpstrid": "4ef00bbec7020af2" + } + ], + "none": [ + { + "value": "iakoute", + "xpstrid": "79262aa5d69820da" + }, + { + "value": "hanifi", + "xpstrid": "e8cecebdded8d76" + }, + { + "value": "Monde", + "xpstrid": "73c7c09de32184" + } + ] + */ + setHidden(hiddenObject) { + this.hiddenObject = hiddenObject; + } + + itemIsChecked(e) { + if (!this.hiddenObject[e.subtype]) { + return false; + } + const pathValueArray = this.hiddenObject[e.subtype]; + return pathValueArray.some( + (p) => p.xpstrid === e.xpstrid && p.value === e.winning + ); + } + + /** + * After the user has voted, reset attributes for the entry that might have + * changed or disappeared as a result of voting. Also update the totals for + * this DashData. + */ + cleanEntry(dashEntry) { + this.removeEntryCats(dashEntry); + dashEntry.clean(); + } + + removeEntry(dashEntry) { + this.removeEntryCats(dashEntry); + const index = this.entries.indexOf(dashEntry); + this.entries.splice(index, 1); + } + + removeEntryCats(dashEntry) { + dashEntry.cats.forEach((cat) => { + this.catSize[cat]--; + if (!this.catSize[cat]) { + this.cats.delete(cat); + delete this.catSize[cat]; + } + if (this.catFirst[cat] === dashEntry.xpstrid) { + this.findCatFirst(cat); + } + }); + } + + findCatFirst(cat) { + for (let dashEntry of this.entries) { + if (dashEntry.cat === cat) { + this.catFirst[cat] = dashEntry; + return; + } + } + delete this.catFirst[cat]; + } +} + +class DashEntry { + /** + * Construct a new DashEntry object + * + * @param {String} xpstrid the xpath hex string id like "710b6e70773e5764" + * @param {String} code the code like "long-one-nominative" + * @param {String} english the English value like "{0} metric pint" + * + * @returns the new DashEntry object + */ + constructor(xpstrid, code, english) { + this.xpstrid = xpstrid; + this.code = code; + this.english = english; + this.cats = new Set(); // set of notification category names + this.winning = null; + this.previousEnglish = null; + this.comment = null; + this.subtype = null; + this.checked = false; + } + + setSectionPageHeader(section, page, header) { + this.section = section; // e.g., "Units" + this.page = page; // e.g., "Volume" + this.header = header; // e.g., "pint-metric" + } + + setWinning(winning) { + this.winning = winning; // like "{0} pinte métrique" + } + + setPreviousEnglish(previousEnglish) { + this.previousEnglish = previousEnglish; // e.g., "{0} metric pint" + } + + setComment(comment) { + this.comment = comment; // e.g., "<missing placeholders> Need at least 1 placeholder(s), but only have 0. Placeholders..." + } + + setSubtype(subtype) { + this.subtype = subtype; // e.g., "missingPlaceholders" + } + + setChecked(checked) { + this.checked = checked; // boolean; the user added a checkmark for this entry + } + + addCategory(category) { + this.cats.add(category); // e.g., "Error", "Disputed", "English_Changed + } + + /** + * After the user has voted, reset attributes that might have changed + * or disappeared as a result of voting. Any updated values for these + * attributes will be added based on the new server response. + */ + clean() { + this.cats = new Set(); + this.winning = null; + this.comment = null; + this.subtype = null; + } +} + +let fetchErr = ""; + +let viewSetDataCallback = null; + +function doFetch(callback) { + viewSetDataCallback = callback; + fetchErr = ""; + const locale = cldrStatus.getCurrentLocale(); + const level = cldrCoverage.effectiveName(locale); + if (!locale || !level) { + fetchErr = "Please choose a locale and a coverage level first."; + return; + } + const url = `api/summary/dashboard/${locale}/${level}`; + cldrAjax + .doFetch(url) + .then(cldrAjax.handleFetchErrors) + .then((data) => data.json()) + .then(setData) + .catch((err) => { + const msg = "Error loading Dashboard data: " + err; + console.error(msg); + fetchErr = msg; + }); +} + +function getFetchError() { + return fetchErr; +} /** - * Set the data for the Dashboard, add "total" and "checked" fields, and index it. + * Set the data for the Dashboard, converting from json to a DashData object * * The json data as received from the back end is ordered by category, then by section, page, header, code, ... - * (but those are not ordered alphabetically). It is presented to the user in that same order. + * (but those are not ordered alphabetically). * - * @param data - an object with these elements: + * @param json - an object with these elements: * notifications - an array of objects (locally named "catData" meaning "all the data for one category"), * each having these elements: * category - a string like "Error" or "English_Changed" - * total - an integer (number of entries in this category), added by addCounts, not in json * groups - an array of objects, each having these elements: * header - a string * page - a string @@ -52,246 +301,57 @@ let pathIndex = {}; * Dashboard only uses data.notifications. There are additional fields * data.* used by cldrProgress for progress meters. * - * @return the modified data (with totals, etc., added) + * @return the modified/reorganized data as a DashData object */ -function setData(data) { - cldrProgress.updateVoterCompletion(data); - addCounts(data); - makePathIndex(data); - return data; -} - -/** - * Calculate total counts; modify data by adding "total" for each category - * - * @param data - */ -function addCounts(data) { - for (let catData of data.notifications) { - catData.total = 0; - for (let group of catData.groups) { - catData.total += group.entries.length; - } - } -} - -/** - * Create the index; also set checked = true/false for all entries - * - * @param data - */ -function makePathIndex(data) { - pathIndex = {}; - for (let catData of data.notifications) { - for (let group of catData.groups) { - for (let entry of group.entries) { - entry.checked = itemIsChecked(data, entry); - if (!pathIndex[entry.xpstrid]) { - pathIndex[entry.xpstrid] = {}; - } else if (pathIndex[entry.xpstrid][catData.category]) { - console.error( - "Duplicate in makePathIndex: " + - entry.xpstrid + - ", " + - catData.category - ); - } - pathIndex[entry.xpstrid][catData.category] = entry; - } - } +function setData(json) { + cldrProgress.updateVoterCompletion(json); + const newData = convertData(json); + if (viewSetDataCallback) { + viewSetDataCallback(newData); } + return newData; } -function itemIsChecked(data, entry) { - if (!data.hidden || !data.hidden[entry.subtype]) { - return false; - } - const pathValueArray = data.hidden[entry.subtype]; - return pathValueArray.some( - (p) => p.xpstrid === entry.xpstrid && p.value === entry.winning - ); +function convertData(json) { + const newData = new DashData(); + newData.setHidden(json.hidden); + newData.addEntriesFromJson(json.notifications); + return newData; } /** * A user has voted. Update the Dashboard data and index as needed. * * Even though the json is only for one path, it may have multiple notifications, - * with different categories such as "Warning" and "English_Changed", - * affecting multiple Dashboard rows. + * with different categories such as "Warning" and "English_Changed". * * Ensure that the data gets updated for (1) each new or modified notification, * and (2) each obsolete notification -- if a notification for this path occurs in * the (old) data but not in the (new) json, it's obsolete and must be removed. * - * @param data - the Dashboard data for all paths, to be updated + * @param dashData - the DashData (new format) for all paths, to be updated * @param json - the response to a request by cldrTable.refreshSingleRow, - * containing notifications for a single path + * containing notifications for a single path (old format) */ -function updatePath(data, json) { - try { - const updater = newPathUpdater(data, json); - updater.oldCategories.forEach((category) => { - if (updater.newCategories.includes(category)) { - updateEntry(updater, category); - } else { - removeEntry(updater, category); - } - }); - updater.newCategories.forEach((category) => { - if (!updater.oldCategories.includes(category)) { - addEntry(updater, category); - } - }); - } catch (e) { - cldrNotify.exception(e, "updating path for Dashboard"); - } - return data; // for unit test -} - -function newPathUpdater(data, json) { - if (!json.xpstrid) { - cldrNotify.error( - "Invalid server response", - "Missing path identifier for Dashboard" - ); - return null; - } - const updater = { - data: data, - json: json, - xpstrid: json.xpstrid, - oldCategories: [], - newCategories: [], - group: null, - }; - for (let catData of json.notifications) { - updater.newCategories.push(catData.category); - } - updater.newCategories = updater.newCategories.sort(); - if (updater.xpstrid in pathIndex) { - updater.oldCategories = Object.keys(pathIndex[updater.xpstrid]).sort(); - } - return updater; -} - -function updateEntry(updater, category) { - try { - const catData = getDataForCategory(updater.data, category); - for (let group of catData.groups) { - const entries = group.entries; - for (let i in entries) { - if (entries[i].xpstrid === updater.xpstrid) { - const newEntry = getNewEntry(updater, category); - pathIndex[updater.xpstrid][category] = entries[i] = newEntry; - return; - } - } - } - } catch (e) { - cldrNotify.exception(e, "updating dashboard entry"); - } -} - -function removeEntry(updater, category) { +function updatePath(dashData, json) { try { - const catData = getDataForCategory(updater.data, category); - for (let group of catData.groups) { - const entries = group.entries; - for (let i in entries) { - if (entries[i].xpstrid === updater.xpstrid) { - entries.splice(i, 1); - --catData.total; - delete pathIndex[updater.xpstrid][category]; - return; - } + if (json.xpstrid in dashData.pathIndex) { + // We already have an entry for this path + const dashEntry = dashData.pathIndex[json.xpstrid]; + if (!json.notifications?.length) { + // The path no longer has any notifications, so remove the entry + dashData.removeEntry(dashEntry); + return dashData; // for unit test } + // Clear attributes that might have changed or disappeared as a result of voting. + // They will be updated/restored from the json. + dashData.cleanEntry(dashEntry); } + dashData.addEntriesFromJson(json.notifications); } catch (e) { - cldrNotify.exception(e, "removing dashboard entry"); - } -} - -function addEntry(updater, category) { - try { - const newEntry = getNewEntry(updater, category); // sets updater.group - const catData = getDataForCategory(updater.data, category); - const group = getMatchingGroup(catData, updater.group); - // TODO: insert in a particular order; see https://unicode-org.atlassian.net/browse/CLDR-15202 - group.entries.push(newEntry); - catData.total++; - if (!(updater.xpstrid in pathIndex)) { - pathIndex[updater.xpstrid] = {}; - } - pathIndex[updater.xpstrid][category] = newEntry; - } catch (e) { - cldrNotify.exception(e, "adding dashboard entry"); - } -} - -function getNewEntry(updater, category) { - for (let catData of updater.json.notifications) { - if (catData.category === category) { - for (let group of catData.groups) { - for (let entry of group.entries) { - if (entry.xpstrid === updater.xpstrid) { - updater.group = group; - entry.checked = itemIsChecked(updater.data, entry); - return entry; - } - } - } - } - } - throw new Error("New entry not found"); -} - -function getMatchingGroup(catData, groupToMatch) { - if (!groupToMatch) { - throw new Error("Matching dashboard group not found"); - } - for (let group of catData.groups) { - if (groupsMatch(group, groupToMatch)) { - return group; - } - } - const newGroup = cloneGroup(groupToMatch); - // TODO: insert in a particular order; see https://unicode-org.atlassian.net/browse/CLDR-15202 - catData.groups.push(newGroup); - return newGroup; -} - -function groupsMatch(groupA, groupB) { - return ( - groupA.header === groupB.header && - groupA.page === groupB.page && - groupA.section === groupB.section - ); -} - -function cloneGroup(group) { - const newGroup = { - header: group.header, - page: group.page, - section: group.section, - entries: [], - }; - return newGroup; -} - -function getDataForCategory(data, category) { - for (let catData of data.notifications) { - if (catData.category === category) { - return catData; - } + cldrNotify.exception(e, "updating path for Dashboard"); } - const newCatData = { - category: category, - total: 0, - groups: [], - }; - // TODO: insert in a particular order; see https://unicode-org.atlassian.net/browse/CLDR-15202 - data.notifications.push(newCatData); - return newCatData; + return dashData; // for unit test } /** @@ -322,6 +382,7 @@ function getCheckmarkUrl(entry, locale) { } /** + * Download as XLSX spreadsheet * * @param {Object} data processed data to download * @param {String} locale locale id @@ -329,20 +390,16 @@ function getCheckmarkUrl(entry, locale) { */ async function downloadXlsx(data, locale, cb) { const xpathMap = cldrSurvey.getXpathMap(); - const { coverageLevel, notifications } = data; + const { coverageLevel, entries } = data; // Fetch all XPaths in parallel since it'll take a while cb(`Loading…`); const allXpaths = []; - for (const { groups } of notifications) { - for (const { section, entries } of groups) { - if (section === "Reports") { - continue; // skip this - } - for (const { xpstrid } of entries) { - allXpaths.push(xpstrid); - } + for (let dashEntry of entries) { + if (dashEntry.section === "Reports") { + continue; // skip this } + allXpaths.push(dashEntry.xpstrid); } await Promise.all(allXpaths.map((x) => xpathMap.get(x))); cb(`Calculating…`); @@ -366,29 +423,25 @@ async function downloadXlsx(data, locale, cb) { ], ]; - for (const { category, groups } of notifications) { - for (const { header, page, section, entries } of groups) { - for (const { code, english, old, subtype, winning, xpstrid } of entries) { - const xpath = - section === "Reports" ? "-" : (await xpathMap.get(xpstrid)).path; - ws_data.push([ - category, - header, - page, - section, - code, - english, - old, - subtype, - winning, - xpstrid, - xpath, - `https://st.unicode.org/cldr-apps/v#/${locale}/${page}/${xpstrid}`, - ]); - } - } + for (let e of entries) { + const xpath = + section === "Reports" ? "-" : (await xpathMap.get(xpstrid)).path; + const url = `https://st.unicode.org/cldr-apps/v#/${e.locale}/${e.page}/${e.xpstrid}`; + ws_data.push([ + e.cat, + e.header, + e.page, + e.section, + e.code, + e.english, + e.old, + e.subtype, + e.winning, + e.xpstrid, + xpath, + url, + ]); } - const ws = XLSX.utils.aoa_to_sheet(ws_data); // cldrXlsx.pushComment(ws, "C1", `As of ${new Date().toISOString()}`); const wb = XLSX.utils.book_new(); @@ -402,4 +455,11 @@ async function downloadXlsx(data, locale, cb) { cb(null); } -export { saveEntryCheckmark, setData, updatePath, downloadXlsx }; +export { + doFetch, + getFetchError, + saveEntryCheckmark, + setData, + updatePath, + downloadXlsx, +}; diff --git a/tools/cldr-apps/js/src/esm/cldrTable.mjs b/tools/cldr-apps/js/src/esm/cldrTable.mjs index 51d723bba76..d983f105687 100644 --- a/tools/cldr-apps/js/src/esm/cldrTable.mjs +++ b/tools/cldr-apps/js/src/esm/cldrTable.mjs @@ -26,7 +26,7 @@ import * as cldrXPathUtils from "./cldrXpathUtils.mjs"; const HEADER_ID_PREFIX = "header_"; const ROW_ID_PREFIX = "row_"; // formerly "r@" -const CLDR_TABLE_DEBUG = true; +const CLDR_TABLE_DEBUG = false; /* * NO_WINNING_VALUE indicates the server delivered path data without a valid winning value. diff --git a/tools/cldr-apps/js/src/esm/cldrVote.mjs b/tools/cldr-apps/js/src/esm/cldrVote.mjs index 81ce9e80c78..13ca7c72a46 100644 --- a/tools/cldr-apps/js/src/esm/cldrVote.mjs +++ b/tools/cldr-apps/js/src/esm/cldrVote.mjs @@ -13,7 +13,7 @@ import * as cldrSurvey from "./cldrSurvey.mjs"; import * as cldrTable from "./cldrTable.mjs"; import * as cldrText from "./cldrText.mjs"; -const CLDR_VOTE_DEBUG = true; +const CLDR_VOTE_DEBUG = false; /** * The special "vote level" selected by the user, or zero for default. diff --git a/tools/cldr-apps/js/src/views/DashboardWidget.vue b/tools/cldr-apps/js/src/views/DashboardWidget.vue index 3100513b303..67f6c422a48 100644 --- a/tools/cldr-apps/js/src/views/DashboardWidget.vue +++ b/tools/cldr-apps/js/src/views/DashboardWidget.vue @@ -29,15 +29,27 @@ > ↻ - -