diff --git a/User Guide.md b/User Guide.md index 4c64e573..07a4fe08 100644 --- a/User Guide.md +++ b/User Guide.md @@ -40,6 +40,7 @@ The fundamental unit of a roadmap is a project milestone. In the context of road - Roadmaps are represented by a single root node, or GitHub issue, which contains links to milestones contained within that roadmap. - The roadmap root node and child milestones can be in any public repository as long as the issues satisfy the requirements outlined in this document. - This means that you can link to existing GitHub issues as child milestones. +- Errors will be logged and displayed for the user in starmap.site ### Milestone Encodings @@ -93,18 +94,38 @@ can expect to get from this milestone. #### Children - A milestone may have child milestones. -- Child milestones are simply full URL links to other GitHub milestone issues. -- Child milestones can exist in any public Github repository. +- Child milestones are simply GitHub issue identifiers (#, /#, or full URLs) to other GitHub milestone issues. +- Child milestones can exist in any public GitHub repository. - It is expected that child milestone issues are themselves properly encoded milestones; otherwise they will be ignored by Starmap. - Within a parent issue, child milestones are encoded as follows (raw Markdown): +##### Tasklist syntax + +Tasklists allow for "taskifying" of strings, and we have no way to link a random string to a GitHub issue. You must convert any [tasks to issues](https://docs.github.com/en/issues/tracking-your-work-with-issues/about-tasklists#converting-draft-issues-to-issues-in-a-tasklist) for them to show up as a child milestone. + +We will do our best to support the expected syntax of GitHub's tasklist functionality. + +See https://docs.github.com/en/issues/tracking-your-work-with-issues/about-tasklists#creating-tasklists and https://github.com/pln-planning-tools/Starmap/issues/245 for more details. + +``` +```[tasklist] +### Tasks +- [ ] https://github.com/pln-roadmap/Roadmap-Vizualizer/issues/10 +- [ ] https://github.com/pln-roadmap/Roadmap-Vizualizer/issues/9 +- [ ] https://github.com/pln-roadmap/Roadmap-Vizualizer/issues/8 +\``` +``` + +##### "Children:" syntax + +This syntax is deprecated. Please see https://github.com/pln-planning-tools/Starmap/issues/245 for more details. + ``` Children: - https://github.com/pln-roadmap/Roadmap-Vizualizer/issues/10 - https://github.com/pln-roadmap/Roadmap-Vizualizer/issues/9 - https://github.com/pln-roadmap/Roadmap-Vizualizer/issues/8 ``` -- Errors will be logged and displayed for the user in starmap.site ### Progress Indicators @@ -127,6 +148,39 @@ Children: ### Templates #### Root Node Issue + +##### Using GitHub Tasklists + +``` +Title: [Team/Project Name] [Duration] Roadmap + +Description (optional): +The goal of this roadmap is to outline the key milestones and deliverables for our team/project over the next [Duration]. + +```[tasklist] +### Any descriptor or other text +- [ ] #123 +- [ ] org/repo#123 +- [ ] some non-link description +- [ ] https://github.com/org/repo/issue/987 + +### Any text +- [ ] #456 +- [ ] org/repo#567 +- [ ] https://github.com/other-org/other-repo/issue/987 + +\``` + +Note: This roadmap is subject to change as priorities and circumstances evolve. + +Starmap Link: [Starmap Link] + +``` + +##### Using "Children:" + +**NOTE:** The children: section is deprecated. Please see https://github.com/pln-planning-tools/Starmap/issues/245 for more details + ``` Title: [Team/Project Name] [Duration] Roadmap diff --git a/lib/backend/getGithubIssueDataWithGroupAndChildren.ts b/lib/backend/getGithubIssueDataWithGroupAndChildren.ts index d9dfe4e0..b0a76788 100644 --- a/lib/backend/getGithubIssueDataWithGroupAndChildren.ts +++ b/lib/backend/getGithubIssueDataWithGroupAndChildren.ts @@ -5,7 +5,7 @@ import { resolveChildren } from './resolveChildren'; import { resolveChildrenWithDepth } from './resolveChildrenWithDepth'; export async function getGithubIssueDataWithGroupAndChildren (issueData: GithubIssueDataWithGroup, errorManager: ErrorManager, usePendingChildren = false): Promise { - const childrenParsed: ParserGetChildrenResponse[] = getChildren(issueData.body_html); + const childrenParsed: ParserGetChildrenResponse[] = getChildren(issueData); let pendingChildren: PendingChildren[] | undefined = undefined; let children: GithubIssueDataWithGroupAndChildren[] = []; diff --git a/lib/backend/issue.ts b/lib/backend/issue.ts index 03843883..0547e999 100644 --- a/lib/backend/issue.ts +++ b/lib/backend/issue.ts @@ -27,6 +27,7 @@ export async function getIssue ({ owner, repo, issue_number }): Promise (typeof label !== 'string' ? label.name : label)) as string[], }; diff --git a/lib/parser.ts b/lib/parser.ts index 8bda8390..efdfc423 100644 --- a/lib/parser.ts +++ b/lib/parser.ts @@ -1,7 +1,9 @@ import { parseHTML } from 'linkedom'; import { ErrorManager } from './backend/errorManager'; +import { getValidUrlFromInput } from './getValidUrlFromInput'; import { getEtaDate, isValidChildren } from './helpers'; -import { GithubIssueDataWithChildren, ParserGetChildrenResponse } from './types'; +import { paramsFromUrl } from './paramsFromUrl'; +import { GithubIssueData, GithubIssueDataWithChildren, ParserGetChildrenResponse } from './types'; export const getDueDate = (issue: Pick, errorManager: ErrorManager) => { const { body_html: issueBodyHtml } = issue; @@ -27,8 +29,109 @@ export const getDueDate = (issue: Pick { - const { document } = parseHTML(issue); +function getSectionLines(text: string, sectionHeader: string) { + const sectionIndex = text.indexOf(sectionHeader); + if (sectionIndex === -1) { + return []; + } + const lines = text.substring(sectionIndex).split(/[\r\n]+/).slice(1); + return lines; +} + +const splitAndGetLastItem = (line: string) => line.trim().split(' ').slice(-1)[0] +const ensureTaskListChild = (line: string) => line.trim().indexOf('-') === 0 + +function getUrlStringForChildrenLine(line: string, issue: Pick) { + if (/^#\d+$/.test(line)) { + const { owner, repo } = paramsFromUrl(issue.html_url) + line = `${owner}/${repo}${line}` + } + return getValidUrlFromInput(line).href +} +/** + * We attempt to parse the issue.body for children included in 'tasklist' format + * @see https://github.com/pln-planning-tools/Starmap/issues/245 + * + * @param {string} issue_body + */ +function getChildrenFromTaskList(issue: Pick): ParserGetChildrenResponse[] { + // tasklists require the checkbox style format to recognize children + const lines = getSectionLines(issue.body, '```[tasklist]') + .filter(ensureTaskListChild) + .map(splitAndGetLastItem) + .filter(Boolean); + + if (lines.length === 0) { + throw new Error('Section missing or has no children') + } + + const children: ParserGetChildrenResponse[] = lines.map((line) => ({ + group: 'tasklist', + html_url: getUrlStringForChildrenLine(line, issue), + })); + return children +} + +/** + * A new version of getchildren which parses the issue body_text instead of issue body_html + * + * This function must support recognizing the current issue's organization and repo, because some children may simply be "#43" instead of a github short-id such as "org/repo#43" + * @param {string} issue + */ +function getChildrenNew(issue: Pick): ParserGetChildrenResponse[] { + + try { + return getChildrenFromTaskList(issue); + } catch (e) { + // Could not find children using new tasklist format, + // try to look for "children:" section + } + const lines = getSectionLines(issue.body, 'children:').map((line) => line.trim().split(' ').slice(-1)[0]).filter(Boolean); + if (lines.length === 0) { + throw new Error('Section missing or has no children') + } + + // guard against HTML tags (covers cases where this method is called with issue.body_html instead of issue.body_text) + if (lines.some((line) => line.startsWith('<'))) { + throw new Error('HTML tags found in body_text'); + } + + const children: ParserGetChildrenResponse[] = [] + + for (let i = 0; i < lines.length; i++) { + const currentLine = lines[i] + if (currentLine.length <= 0) { + if (children.length === 0) { + // skip empty lines between children header and children + continue + } else { + // end of children if empty line is found and children is not empty + break + } + } + + try { + children.push({ + group: 'children:', + html_url: getUrlStringForChildrenLine(currentLine, issue) + }) + } catch { + // invalid URL or child issue identifier, exit and return what we have + break + } + } + + return children + +} + +export const getChildren = (issue: Pick): ParserGetChildrenResponse[] => { + try { + return getChildrenNew(issue); + } catch (e) { + // ignore failures for now, fallback to old method. + } + const { document } = parseHTML(issue.body_html); const ulLists = [...document.querySelectorAll('ul')]; const filterListByTitle = (ulLists) => ulLists.filter((list) => { diff --git a/lib/types.d.ts b/lib/types.d.ts index 3627ceb5..103d5e30 100644 --- a/lib/types.d.ts +++ b/lib/types.d.ts @@ -4,6 +4,7 @@ import type { RoadmapMode, IssueStates, DateGranularityState } from './enums' export interface GithubIssueData { body_html: string; + body: string; html_url: string; labels: string[]; node_id: string; diff --git a/pages/api/roadmap.ts b/pages/api/roadmap.ts index 890e517a..9aff499d 100644 --- a/pages/api/roadmap.ts +++ b/pages/api/roadmap.ts @@ -33,7 +33,7 @@ export default async function handler( try { const rootIssue = await getIssue({ owner, repo, issue_number }); - const childrenFromBodyHtml = (!!rootIssue && rootIssue.body_html && getChildren(rootIssue.body_html)) || null; + const childrenFromBodyHtml = (!!rootIssue && rootIssue.body_html && getChildren(rootIssue)) || null; let children: Awaited> = []; try { if (childrenFromBodyHtml != null) { diff --git a/tests/unit/parser.test.ts b/tests/unit/parser.test.ts new file mode 100644 index 00000000..9e7e3001 --- /dev/null +++ b/tests/unit/parser.test.ts @@ -0,0 +1,136 @@ +import { getChildren } from '../../lib/parser'; + +/** + * Test data obtained from calling getIssue() on github.com/protocol/engres/issues/5 on 2023-02-10 @ 5pm PST + */ +const example_body_html = '

children:

\n\n

eta: 2023Q2

\n

View in https://www.starmaps.app/roadmap/github.com/protocol/engres/issues/5#simple

' +const example_body_text = 'children:\n\nprotocol/bedrock#5\nprotocol/bedrock#11\nfilecoin-project/ref-fvm#1144\nfilecoin-project/ref-fvm#1143\nprotocol/netops#34\nprotocol/netops#47\ndrand/roadmap#8\ndrand/roadmap#12\nprotocol/ConsensusLab#180\nprotocol/ConsensusLab#185\nprotocol/ConsensusLab#186\nfilecoin-station/roadmap#3\nfilecoin-station/roadmap#10\nfilecoin-saturn/roadmap#1\nfilecoin-station/roadmap#4\nfilecoin-saturn/roadmap#2\ncryptonetlab/roadmap#19\n\neta: 2023Q2\nView in https://www.starmaps.app/roadmap/github.com/protocol/engres/issues/5#simple' +const example_body = 'children: \r\n- https://github.com/protocol/bedrock/issues/5\r\n- https://github.com/protocol/bedrock/issues/11\r\n- https://github.com/filecoin-project/ref-fvm/issues/1144\r\n- https://github.com/filecoin-project/ref-fvm/issues/1143\r\n- https://github.com/protocol/netops/issues/34\r\n- https://github.com/protocol/netops/issues/47\r\n- https://github.com/drand/roadmap/issues/8\r\n- https://github.com/drand/roadmap/issues/12\r\n- https://github.com/protocol/ConsensusLab/issues/180\r\n- https://github.com/protocol/ConsensusLab/issues/185\r\n- https://github.com/protocol/ConsensusLab/issues/186\r\n- https://github.com/filecoin-station/roadmap/issues/3\r\n- https://github.com/filecoin-station/roadmap/issues/10\r\n- https://github.com/filecoin-saturn/roadmap/issues/1\r\n- https://github.com/filecoin-station/roadmap/issues/4\r\n- https://github.com/filecoin-saturn/roadmap/issues/2\r\n- https://github.com/cryptonetlab/roadmap/issues/19\r\n\r\neta: 2023Q2\r\n\r\nView in https://www.starmaps.app/roadmap/github.com/protocol/engres/issues/5#simple\r\n' + +/** + * Test data obtained from calling getIssue() on github.com/ipfs/ipfs-gui/issues/106 on 2023-02-10 @ 5:52pm PST + */ +const example_tasklist_body = 'eta: 2023Q4\r\n\r\nchildren:\r\n- [ ] #121\r\n- [ ] #122\r\n- [ ] #123\r\n- [ ] #124\r\n\r\n```[tasklist]\r\n### Tasks\r\n- [ ] #121\r\n- [ ] #122\r\n- [ ] #123\r\n- [ ] #124\r\n```\r\n' + +/** + * Test data manually removing "children:" from the above example_tasklist_body + */ +const example_tasklist_body_only = 'eta: 2023Q4\r\n\r\n```[tasklist]\r\n### Tasks\r\n- [ ] #121\r\n- [ ] #122\r\n- [ ] #123\r\n- [ ] #124\r\n```\r\n' + +const expectedResult = [ + { + group: 'children:', + html_url: 'https://github.com/protocol/bedrock/issues/5' + }, + { + group: 'children:', + html_url: 'https://github.com/protocol/bedrock/issues/11', + }, + { + group: 'children:', + html_url: 'https://github.com/filecoin-project/ref-fvm/issues/1144', + }, + { + group: 'children:', + html_url: 'https://github.com/filecoin-project/ref-fvm/issues/1143', + }, + { + group: 'children:', + html_url: 'https://github.com/protocol/netops/issues/34', + }, + { + group: 'children:', + html_url: 'https://github.com/protocol/netops/issues/47', + }, + { + group: 'children:', + html_url: 'https://github.com/drand/roadmap/issues/8', + }, + { + group: 'children:', + html_url: 'https://github.com/drand/roadmap/issues/12', + }, + { + group: 'children:', + html_url: 'https://github.com/protocol/ConsensusLab/issues/180', + }, + { + group: 'children:', + html_url: 'https://github.com/protocol/ConsensusLab/issues/185', + }, + { + group: 'children:', + html_url: 'https://github.com/protocol/ConsensusLab/issues/186', + }, + { + group: 'children:', + html_url: 'https://github.com/filecoin-station/roadmap/issues/3', + }, + { + group: 'children:', + html_url: 'https://github.com/filecoin-station/roadmap/issues/10', + }, + { + group: 'children:', + html_url: 'https://github.com/filecoin-saturn/roadmap/issues/1', + }, + { + group: 'children:', + html_url: 'https://github.com/filecoin-station/roadmap/issues/4', + }, + { + group: 'children:', + html_url: 'https://github.com/filecoin-saturn/roadmap/issues/2', + }, + { + group: 'children:', + html_url: 'https://github.com/cryptonetlab/roadmap/issues/19', + }, +] + +describe('parser', function() { + describe('getChildren', function() { + it('Can parse children from issue.body_html', function() { + const children = getChildren({ body_html: example_body_html, body: '', html_url: '' }); + expect(Array.isArray(children)).toBe(true); + expect(children).toHaveLength(17); + expect(children).toStrictEqual(expectedResult) + }) + it('Can parse children from issue.body_text', function() { + const children = getChildren({ body_html: '', body: example_body_text, html_url: '' }); + expect(Array.isArray(children)).toBe(true); + expect(children).toHaveLength(17); + expect(children).toStrictEqual(expectedResult) + }) + it('Can parse children from issue.body', function() { + const children = getChildren({ body_html: '', body: example_body, html_url: '' }); + expect(Array.isArray(children)).toBe(true); + expect(children).toHaveLength(17); + expect(children).toStrictEqual(expectedResult) + }) + + it('Can parse tasklist children from body that also has children:', function() { + const children = getChildren({ body_html: '', body: example_tasklist_body, html_url: 'https://github.com/ipfs/ipfs-gui/issues/106' }); + expect(Array.isArray(children)).toBe(true); + expect(children).toHaveLength(4); + expect(children).toStrictEqual([ + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/121' }, + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/122' }, + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/123' }, + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/124' } + ]) + }) + + it('Can parse tasklist children from body', function() { + const children = getChildren({ body_html: '', body: example_tasklist_body_only, html_url: 'https://github.com/ipfs/ipfs-gui/issues/106' }); + expect(Array.isArray(children)).toBe(true); + expect(children).toHaveLength(4); + expect(children).toStrictEqual([ + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/121' }, + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/122' }, + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/123' }, + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/124' } + ]) + }) + }) +})