From 3088103c3506b71b5f7649ca2d02d67e9b70b196 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 10 Feb 2023 17:08:00 -0800 Subject: [PATCH 1/7] chore: add unit test for getChildren --- tests/unit/parser.test.ts | 87 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 tests/unit/parser.test.ts diff --git a/tests/unit/parser.test.ts b/tests/unit/parser.test.ts new file mode 100644 index 00000000..1f6cb11d --- /dev/null +++ b/tests/unit/parser.test.ts @@ -0,0 +1,87 @@ +import { getChildren } from '../../lib/parser'; + +/** + * Test data obtained from calling getIssue() on github.com/protocol/engres/issues/5 on 2023-02-10 @ 5pm PST + */ +const example_body_html = '

children:

\n\n

eta: 2023Q2

\n

View in https://www.starmaps.app/roadmap/github.com/protocol/engres/issues/5#simple

' +const example_body_text = 'children:\n\nprotocol/bedrock#5\nprotocol/bedrock#11\nfilecoin-project/ref-fvm#1144\nfilecoin-project/ref-fvm#1143\nprotocol/netops#34\nprotocol/netops#47\ndrand/roadmap#8\ndrand/roadmap#12\nprotocol/ConsensusLab#180\nprotocol/ConsensusLab#185\nprotocol/ConsensusLab#186\nfilecoin-station/roadmap#3\nfilecoin-station/roadmap#10\nfilecoin-saturn/roadmap#1\nfilecoin-station/roadmap#4\nfilecoin-saturn/roadmap#2\ncryptonetlab/roadmap#19\n\neta: 2023Q2\nView in https://www.starmaps.app/roadmap/github.com/protocol/engres/issues/5#simple' + +describe('parser', function() { + describe('getChildren', function() { + it('Can parse children from body_html', function() { + const children = getChildren(example_body_html); + expect(Array.isArray(children)).toBe(true); + expect(children).toHaveLength(17); + expect(children).toStrictEqual([ + { + group: 'children:', + html_url: 'https://github.com/protocol/bedrock/issues/5' + }, + { + "group": "children:", + "html_url": "https://github.com/protocol/bedrock/issues/11", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-project/ref-fvm/issues/1144", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-project/ref-fvm/issues/1143", + }, + { + "group": "children:", + "html_url": "https://github.com/protocol/netops/issues/34", + }, + { + "group": "children:", + "html_url": "https://github.com/protocol/netops/issues/47", + }, + { + "group": "children:", + "html_url": "https://github.com/drand/roadmap/issues/8", + }, + { + "group": "children:", + "html_url": "https://github.com/drand/roadmap/issues/12", + }, + { + "group": "children:", + "html_url": "https://github.com/protocol/ConsensusLab/issues/180", + }, + { + "group": "children:", + "html_url": "https://github.com/protocol/ConsensusLab/issues/185", + }, + { + "group": "children:", + "html_url": "https://github.com/protocol/ConsensusLab/issues/186", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-station/roadmap/issues/3", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-station/roadmap/issues/10", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-saturn/roadmap/issues/1", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-station/roadmap/issues/4", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-saturn/roadmap/issues/2", + }, + { + "group": "children:", + "html_url": "https://github.com/cryptonetlab/roadmap/issues/19", + }, + ]) + }) + }) +}) From b075a90e524aa600571d8446907d316de40ebbd6 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 10 Feb 2023 17:48:31 -0800 Subject: [PATCH 2/7] chore: support parsing issue body_text --- lib/parser.ts | 57 ++++++++++++++- tests/unit/parser.test.ts | 150 ++++++++++++++++++++------------------ 2 files changed, 134 insertions(+), 73 deletions(-) diff --git a/lib/parser.ts b/lib/parser.ts index 8bda8390..e9a829b1 100644 --- a/lib/parser.ts +++ b/lib/parser.ts @@ -1,5 +1,6 @@ import { parseHTML } from 'linkedom'; import { ErrorManager } from './backend/errorManager'; +import { getValidUrlFromInput } from './getValidUrlFromInput'; import { getEtaDate, isValidChildren } from './helpers'; import { GithubIssueDataWithChildren, ParserGetChildrenResponse } from './types'; @@ -27,8 +28,60 @@ export const getDueDate = (issue: Pick { - const { document } = parseHTML(issue); +/** + * A new version of getchildren which parses the issue body_text instead of issue body_html + * + * This function must support recognizing the current issue's organization and repo, because some children may simply be "#43" instead of a github short-id such as "org/repo#43" + * @param {string} issue + */ +export function getChildrenNew(issue_text: string): ParserGetChildrenResponse[] { + // first we need to ensure that the issue contains "children: " in the body + const childrenIndex = issue_text.indexOf('children:'); + if (childrenIndex === -1) { + throw new Error('No children found in body_text'); + } + const children: ParserGetChildrenResponse[] = [] + const lines = issue_text.substring(childrenIndex).split(/\n/).slice(1) + + for (let i = 0; i < lines.length; i++) { + const currentLine = lines[i] + if (currentLine.length <= 0) { + if (children.length === 0) { + // skip empty lines between children header and children + continue + } else { + // end of children if empty line is found and children is not empty + break + } + } + // guard against HTML tags (covers cases where this method is called with issue.body_html instead of issue.body_text) + if (currentLine.startsWith('<')) { + throw new Error('HTML tags found in body_text'); + } + + try { + const url = getValidUrlFromInput(currentLine) + children.push({ + group: 'children:', + html_url: url.href + }) + } catch { + // invalid URL or child issue identifier, exit and return what we have + break + } + } + + return children + +} + +export const getChildren = (issue_html: string): ParserGetChildrenResponse[] => { + try { + return getChildrenNew(issue_html); + } catch (e) { + // ignore failures for now, fallback to old method. + } + const { document } = parseHTML(issue_html); const ulLists = [...document.querySelectorAll('ul')]; const filterListByTitle = (ulLists) => ulLists.filter((list) => { diff --git a/tests/unit/parser.test.ts b/tests/unit/parser.test.ts index 1f6cb11d..8f597f31 100644 --- a/tests/unit/parser.test.ts +++ b/tests/unit/parser.test.ts @@ -1,4 +1,4 @@ -import { getChildren } from '../../lib/parser'; +import { getChildren, getChildrenNew } from '../../lib/parser'; /** * Test data obtained from calling getIssue() on github.com/protocol/engres/issues/5 on 2023-02-10 @ 5pm PST @@ -6,82 +6,90 @@ import { getChildren } from '../../lib/parser'; const example_body_html = '

children:

\n\n

eta: 2023Q2

\n

View in https://www.starmaps.app/roadmap/github.com/protocol/engres/issues/5#simple

' const example_body_text = 'children:\n\nprotocol/bedrock#5\nprotocol/bedrock#11\nfilecoin-project/ref-fvm#1144\nfilecoin-project/ref-fvm#1143\nprotocol/netops#34\nprotocol/netops#47\ndrand/roadmap#8\ndrand/roadmap#12\nprotocol/ConsensusLab#180\nprotocol/ConsensusLab#185\nprotocol/ConsensusLab#186\nfilecoin-station/roadmap#3\nfilecoin-station/roadmap#10\nfilecoin-saturn/roadmap#1\nfilecoin-station/roadmap#4\nfilecoin-saturn/roadmap#2\ncryptonetlab/roadmap#19\n\neta: 2023Q2\nView in https://www.starmaps.app/roadmap/github.com/protocol/engres/issues/5#simple' +const expectedResult = [ + { + group: 'children:', + html_url: 'https://github.com/protocol/bedrock/issues/5' + }, + { + "group": "children:", + "html_url": "https://github.com/protocol/bedrock/issues/11", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-project/ref-fvm/issues/1144", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-project/ref-fvm/issues/1143", + }, + { + "group": "children:", + "html_url": "https://github.com/protocol/netops/issues/34", + }, + { + "group": "children:", + "html_url": "https://github.com/protocol/netops/issues/47", + }, + { + "group": "children:", + "html_url": "https://github.com/drand/roadmap/issues/8", + }, + { + "group": "children:", + "html_url": "https://github.com/drand/roadmap/issues/12", + }, + { + "group": "children:", + "html_url": "https://github.com/protocol/ConsensusLab/issues/180", + }, + { + "group": "children:", + "html_url": "https://github.com/protocol/ConsensusLab/issues/185", + }, + { + "group": "children:", + "html_url": "https://github.com/protocol/ConsensusLab/issues/186", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-station/roadmap/issues/3", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-station/roadmap/issues/10", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-saturn/roadmap/issues/1", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-station/roadmap/issues/4", + }, + { + "group": "children:", + "html_url": "https://github.com/filecoin-saturn/roadmap/issues/2", + }, + { + "group": "children:", + "html_url": "https://github.com/cryptonetlab/roadmap/issues/19", + }, +] + describe('parser', function() { describe('getChildren', function() { it('Can parse children from body_html', function() { const children = getChildren(example_body_html); expect(Array.isArray(children)).toBe(true); expect(children).toHaveLength(17); - expect(children).toStrictEqual([ - { - group: 'children:', - html_url: 'https://github.com/protocol/bedrock/issues/5' - }, - { - "group": "children:", - "html_url": "https://github.com/protocol/bedrock/issues/11", - }, - { - "group": "children:", - "html_url": "https://github.com/filecoin-project/ref-fvm/issues/1144", - }, - { - "group": "children:", - "html_url": "https://github.com/filecoin-project/ref-fvm/issues/1143", - }, - { - "group": "children:", - "html_url": "https://github.com/protocol/netops/issues/34", - }, - { - "group": "children:", - "html_url": "https://github.com/protocol/netops/issues/47", - }, - { - "group": "children:", - "html_url": "https://github.com/drand/roadmap/issues/8", - }, - { - "group": "children:", - "html_url": "https://github.com/drand/roadmap/issues/12", - }, - { - "group": "children:", - "html_url": "https://github.com/protocol/ConsensusLab/issues/180", - }, - { - "group": "children:", - "html_url": "https://github.com/protocol/ConsensusLab/issues/185", - }, - { - "group": "children:", - "html_url": "https://github.com/protocol/ConsensusLab/issues/186", - }, - { - "group": "children:", - "html_url": "https://github.com/filecoin-station/roadmap/issues/3", - }, - { - "group": "children:", - "html_url": "https://github.com/filecoin-station/roadmap/issues/10", - }, - { - "group": "children:", - "html_url": "https://github.com/filecoin-saturn/roadmap/issues/1", - }, - { - "group": "children:", - "html_url": "https://github.com/filecoin-station/roadmap/issues/4", - }, - { - "group": "children:", - "html_url": "https://github.com/filecoin-saturn/roadmap/issues/2", - }, - { - "group": "children:", - "html_url": "https://github.com/cryptonetlab/roadmap/issues/19", - }, - ]) + expect(children).toStrictEqual(expectedResult) + }) + it('Can parse children from body_text', function() { + const children = getChildren(example_body_text); + expect(Array.isArray(children)).toBe(true); + expect(children).toHaveLength(17); + expect(children).toStrictEqual(expectedResult) }) }) }) From 18736b630fe5229cd9dca162e7afa80aecf8330e Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 10 Feb 2023 18:47:07 -0800 Subject: [PATCH 3/7] feat: support reading children from tasklists --- .../getGithubIssueDataWithGroupAndChildren.ts | 2 +- lib/backend/issue.ts | 1 + lib/parser.ts | 60 +++++++-- lib/types.d.ts | 1 + pages/api/roadmap.ts | 2 +- tests/unit/parser.test.ts | 115 ++++++++++++------ 6 files changed, 132 insertions(+), 49 deletions(-) diff --git a/lib/backend/getGithubIssueDataWithGroupAndChildren.ts b/lib/backend/getGithubIssueDataWithGroupAndChildren.ts index d9dfe4e0..b0a76788 100644 --- a/lib/backend/getGithubIssueDataWithGroupAndChildren.ts +++ b/lib/backend/getGithubIssueDataWithGroupAndChildren.ts @@ -5,7 +5,7 @@ import { resolveChildren } from './resolveChildren'; import { resolveChildrenWithDepth } from './resolveChildrenWithDepth'; export async function getGithubIssueDataWithGroupAndChildren (issueData: GithubIssueDataWithGroup, errorManager: ErrorManager, usePendingChildren = false): Promise { - const childrenParsed: ParserGetChildrenResponse[] = getChildren(issueData.body_html); + const childrenParsed: ParserGetChildrenResponse[] = getChildren(issueData); let pendingChildren: PendingChildren[] | undefined = undefined; let children: GithubIssueDataWithGroupAndChildren[] = []; diff --git a/lib/backend/issue.ts b/lib/backend/issue.ts index 03843883..0547e999 100644 --- a/lib/backend/issue.ts +++ b/lib/backend/issue.ts @@ -27,6 +27,7 @@ export async function getIssue ({ owner, repo, issue_number }): Promise (typeof label !== 'string' ? label.name : label)) as string[], }; diff --git a/lib/parser.ts b/lib/parser.ts index e9a829b1..199498f8 100644 --- a/lib/parser.ts +++ b/lib/parser.ts @@ -2,7 +2,8 @@ import { parseHTML } from 'linkedom'; import { ErrorManager } from './backend/errorManager'; import { getValidUrlFromInput } from './getValidUrlFromInput'; import { getEtaDate, isValidChildren } from './helpers'; -import { GithubIssueDataWithChildren, ParserGetChildrenResponse } from './types'; +import { paramsFromUrl } from './paramsFromUrl'; +import { GithubIssueData, GithubIssueDataWithChildren, ParserGetChildrenResponse } from './types'; export const getDueDate = (issue: Pick, errorManager: ErrorManager) => { const { body_html: issueBodyHtml } = issue; @@ -28,20 +29,59 @@ export const getDueDate = (issue: Pick): ParserGetChildrenResponse[] { + // tasklists require the checkbox style format to recognize children + const lines = getSectionLines(issue.body, '```[tasklist]').filter((line) => line.trim().indexOf('-') === 0).map((line) => line.trim().split(' ').slice(-1)[0]).filter(Boolean); + if (lines.length === 0) { + throw new Error('Section missing or has no children') + } + const { owner, repo } = paramsFromUrl(issue.html_url) + const children: ParserGetChildrenResponse[] = lines.map((line) => { + if (/^#\d+$/.test(line)) { + line = `${owner}/${repo}${line}` + } + return ({ + group: 'tasklist', + html_url: getValidUrlFromInput(line).href, + }) + }); + return children +} + /** * A new version of getchildren which parses the issue body_text instead of issue body_html * * This function must support recognizing the current issue's organization and repo, because some children may simply be "#43" instead of a github short-id such as "org/repo#43" * @param {string} issue */ -export function getChildrenNew(issue_text: string): ParserGetChildrenResponse[] { - // first we need to ensure that the issue contains "children: " in the body - const childrenIndex = issue_text.indexOf('children:'); - if (childrenIndex === -1) { - throw new Error('No children found in body_text'); +export function getChildrenNew(issue: Pick): ParserGetChildrenResponse[] { + + try { + return getChildrenFromTaskList(issue); + } catch (e) { + // Could not find children using new tasklist format, + // try to look for "children:" section + } + const lines = getSectionLines(issue.body, 'children:').map((line) => line.trim().split(' ').slice(-1)[0]).filter(Boolean); + if (lines.length === 0) { + throw new Error('Section missing or has no children') } const children: ParserGetChildrenResponse[] = [] - const lines = issue_text.substring(childrenIndex).split(/\n/).slice(1) for (let i = 0; i < lines.length; i++) { const currentLine = lines[i] @@ -75,13 +115,13 @@ export function getChildrenNew(issue_text: string): ParserGetChildrenResponse[] } -export const getChildren = (issue_html: string): ParserGetChildrenResponse[] => { +export const getChildren = (issue: Pick): ParserGetChildrenResponse[] => { try { - return getChildrenNew(issue_html); + return getChildrenNew(issue); } catch (e) { // ignore failures for now, fallback to old method. } - const { document } = parseHTML(issue_html); + const { document } = parseHTML(issue.body_html); const ulLists = [...document.querySelectorAll('ul')]; const filterListByTitle = (ulLists) => ulLists.filter((list) => { diff --git a/lib/types.d.ts b/lib/types.d.ts index 3627ceb5..103d5e30 100644 --- a/lib/types.d.ts +++ b/lib/types.d.ts @@ -4,6 +4,7 @@ import type { RoadmapMode, IssueStates, DateGranularityState } from './enums' export interface GithubIssueData { body_html: string; + body: string; html_url: string; labels: string[]; node_id: string; diff --git a/pages/api/roadmap.ts b/pages/api/roadmap.ts index 890e517a..9aff499d 100644 --- a/pages/api/roadmap.ts +++ b/pages/api/roadmap.ts @@ -33,7 +33,7 @@ export default async function handler( try { const rootIssue = await getIssue({ owner, repo, issue_number }); - const childrenFromBodyHtml = (!!rootIssue && rootIssue.body_html && getChildren(rootIssue.body_html)) || null; + const childrenFromBodyHtml = (!!rootIssue && rootIssue.body_html && getChildren(rootIssue)) || null; let children: Awaited> = []; try { if (childrenFromBodyHtml != null) { diff --git a/tests/unit/parser.test.ts b/tests/unit/parser.test.ts index 8f597f31..9e7e3001 100644 --- a/tests/unit/parser.test.ts +++ b/tests/unit/parser.test.ts @@ -1,10 +1,21 @@ -import { getChildren, getChildrenNew } from '../../lib/parser'; +import { getChildren } from '../../lib/parser'; /** * Test data obtained from calling getIssue() on github.com/protocol/engres/issues/5 on 2023-02-10 @ 5pm PST */ const example_body_html = '

children:

\n\n

eta: 2023Q2

\n

View in https://www.starmaps.app/roadmap/github.com/protocol/engres/issues/5#simple

' const example_body_text = 'children:\n\nprotocol/bedrock#5\nprotocol/bedrock#11\nfilecoin-project/ref-fvm#1144\nfilecoin-project/ref-fvm#1143\nprotocol/netops#34\nprotocol/netops#47\ndrand/roadmap#8\ndrand/roadmap#12\nprotocol/ConsensusLab#180\nprotocol/ConsensusLab#185\nprotocol/ConsensusLab#186\nfilecoin-station/roadmap#3\nfilecoin-station/roadmap#10\nfilecoin-saturn/roadmap#1\nfilecoin-station/roadmap#4\nfilecoin-saturn/roadmap#2\ncryptonetlab/roadmap#19\n\neta: 2023Q2\nView in https://www.starmaps.app/roadmap/github.com/protocol/engres/issues/5#simple' +const example_body = 'children: \r\n- https://github.com/protocol/bedrock/issues/5\r\n- https://github.com/protocol/bedrock/issues/11\r\n- https://github.com/filecoin-project/ref-fvm/issues/1144\r\n- https://github.com/filecoin-project/ref-fvm/issues/1143\r\n- https://github.com/protocol/netops/issues/34\r\n- https://github.com/protocol/netops/issues/47\r\n- https://github.com/drand/roadmap/issues/8\r\n- https://github.com/drand/roadmap/issues/12\r\n- https://github.com/protocol/ConsensusLab/issues/180\r\n- https://github.com/protocol/ConsensusLab/issues/185\r\n- https://github.com/protocol/ConsensusLab/issues/186\r\n- https://github.com/filecoin-station/roadmap/issues/3\r\n- https://github.com/filecoin-station/roadmap/issues/10\r\n- https://github.com/filecoin-saturn/roadmap/issues/1\r\n- https://github.com/filecoin-station/roadmap/issues/4\r\n- https://github.com/filecoin-saturn/roadmap/issues/2\r\n- https://github.com/cryptonetlab/roadmap/issues/19\r\n\r\neta: 2023Q2\r\n\r\nView in https://www.starmaps.app/roadmap/github.com/protocol/engres/issues/5#simple\r\n' + +/** + * Test data obtained from calling getIssue() on github.com/ipfs/ipfs-gui/issues/106 on 2023-02-10 @ 5:52pm PST + */ +const example_tasklist_body = 'eta: 2023Q4\r\n\r\nchildren:\r\n- [ ] #121\r\n- [ ] #122\r\n- [ ] #123\r\n- [ ] #124\r\n\r\n```[tasklist]\r\n### Tasks\r\n- [ ] #121\r\n- [ ] #122\r\n- [ ] #123\r\n- [ ] #124\r\n```\r\n' + +/** + * Test data manually removing "children:" from the above example_tasklist_body + */ +const example_tasklist_body_only = 'eta: 2023Q4\r\n\r\n```[tasklist]\r\n### Tasks\r\n- [ ] #121\r\n- [ ] #122\r\n- [ ] #123\r\n- [ ] #124\r\n```\r\n' const expectedResult = [ { @@ -12,84 +23,114 @@ const expectedResult = [ html_url: 'https://github.com/protocol/bedrock/issues/5' }, { - "group": "children:", - "html_url": "https://github.com/protocol/bedrock/issues/11", + group: 'children:', + html_url: 'https://github.com/protocol/bedrock/issues/11', }, { - "group": "children:", - "html_url": "https://github.com/filecoin-project/ref-fvm/issues/1144", + group: 'children:', + html_url: 'https://github.com/filecoin-project/ref-fvm/issues/1144', }, { - "group": "children:", - "html_url": "https://github.com/filecoin-project/ref-fvm/issues/1143", + group: 'children:', + html_url: 'https://github.com/filecoin-project/ref-fvm/issues/1143', }, { - "group": "children:", - "html_url": "https://github.com/protocol/netops/issues/34", + group: 'children:', + html_url: 'https://github.com/protocol/netops/issues/34', }, { - "group": "children:", - "html_url": "https://github.com/protocol/netops/issues/47", + group: 'children:', + html_url: 'https://github.com/protocol/netops/issues/47', }, { - "group": "children:", - "html_url": "https://github.com/drand/roadmap/issues/8", + group: 'children:', + html_url: 'https://github.com/drand/roadmap/issues/8', }, { - "group": "children:", - "html_url": "https://github.com/drand/roadmap/issues/12", + group: 'children:', + html_url: 'https://github.com/drand/roadmap/issues/12', }, { - "group": "children:", - "html_url": "https://github.com/protocol/ConsensusLab/issues/180", + group: 'children:', + html_url: 'https://github.com/protocol/ConsensusLab/issues/180', }, { - "group": "children:", - "html_url": "https://github.com/protocol/ConsensusLab/issues/185", + group: 'children:', + html_url: 'https://github.com/protocol/ConsensusLab/issues/185', }, { - "group": "children:", - "html_url": "https://github.com/protocol/ConsensusLab/issues/186", + group: 'children:', + html_url: 'https://github.com/protocol/ConsensusLab/issues/186', }, { - "group": "children:", - "html_url": "https://github.com/filecoin-station/roadmap/issues/3", + group: 'children:', + html_url: 'https://github.com/filecoin-station/roadmap/issues/3', }, { - "group": "children:", - "html_url": "https://github.com/filecoin-station/roadmap/issues/10", + group: 'children:', + html_url: 'https://github.com/filecoin-station/roadmap/issues/10', }, { - "group": "children:", - "html_url": "https://github.com/filecoin-saturn/roadmap/issues/1", + group: 'children:', + html_url: 'https://github.com/filecoin-saturn/roadmap/issues/1', }, { - "group": "children:", - "html_url": "https://github.com/filecoin-station/roadmap/issues/4", + group: 'children:', + html_url: 'https://github.com/filecoin-station/roadmap/issues/4', }, { - "group": "children:", - "html_url": "https://github.com/filecoin-saturn/roadmap/issues/2", + group: 'children:', + html_url: 'https://github.com/filecoin-saturn/roadmap/issues/2', }, { - "group": "children:", - "html_url": "https://github.com/cryptonetlab/roadmap/issues/19", + group: 'children:', + html_url: 'https://github.com/cryptonetlab/roadmap/issues/19', }, ] describe('parser', function() { describe('getChildren', function() { - it('Can parse children from body_html', function() { - const children = getChildren(example_body_html); + it('Can parse children from issue.body_html', function() { + const children = getChildren({ body_html: example_body_html, body: '', html_url: '' }); + expect(Array.isArray(children)).toBe(true); + expect(children).toHaveLength(17); + expect(children).toStrictEqual(expectedResult) + }) + it('Can parse children from issue.body_text', function() { + const children = getChildren({ body_html: '', body: example_body_text, html_url: '' }); expect(Array.isArray(children)).toBe(true); expect(children).toHaveLength(17); expect(children).toStrictEqual(expectedResult) }) - it('Can parse children from body_text', function() { - const children = getChildren(example_body_text); + it('Can parse children from issue.body', function() { + const children = getChildren({ body_html: '', body: example_body, html_url: '' }); expect(Array.isArray(children)).toBe(true); expect(children).toHaveLength(17); expect(children).toStrictEqual(expectedResult) }) + + it('Can parse tasklist children from body that also has children:', function() { + const children = getChildren({ body_html: '', body: example_tasklist_body, html_url: 'https://github.com/ipfs/ipfs-gui/issues/106' }); + expect(Array.isArray(children)).toBe(true); + expect(children).toHaveLength(4); + expect(children).toStrictEqual([ + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/121' }, + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/122' }, + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/123' }, + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/124' } + ]) + }) + + it('Can parse tasklist children from body', function() { + const children = getChildren({ body_html: '', body: example_tasklist_body_only, html_url: 'https://github.com/ipfs/ipfs-gui/issues/106' }); + expect(Array.isArray(children)).toBe(true); + expect(children).toHaveLength(4); + expect(children).toStrictEqual([ + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/121' }, + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/122' }, + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/123' }, + { group: 'tasklist', html_url: 'https://github.com/ipfs/ipfs-gui/issues/124' } + ]) + }) }) }) From b9e04dc19a9f4bb66195934f0cfb33a67848c88b Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 10 Feb 2023 18:55:58 -0800 Subject: [PATCH 4/7] fix: parsing children: with only #123 identifier --- lib/parser.ts | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/parser.ts b/lib/parser.ts index 199498f8..af3f133b 100644 --- a/lib/parser.ts +++ b/lib/parser.ts @@ -38,6 +38,13 @@ function getSectionLines(text: string, sectionHeader: string) { return lines; } +function getUrlStringForChildrenLine(line: string, issue: Pick) { + if (/^#\d+$/.test(line)) { + const { owner, repo } = paramsFromUrl(issue.html_url) + line = `${owner}/${repo}${line}` + } + return getValidUrlFromInput(line).href +} /** * We attempt to parse the issue.body for children included in 'tasklist' format * @see https://github.com/pln-planning-tools/Starmap/issues/245 @@ -50,16 +57,11 @@ function getChildrenFromTaskList(issue: Pick { - if (/^#\d+$/.test(line)) { - line = `${owner}/${repo}${line}` - } - return ({ + + const children: ParserGetChildrenResponse[] = lines.map((line) => ({ group: 'tasklist', - html_url: getValidUrlFromInput(line).href, - }) - }); + html_url: getUrlStringForChildrenLine(line, issue), + })); return children } @@ -100,10 +102,9 @@ export function getChildrenNew(issue: Pick } try { - const url = getValidUrlFromInput(currentLine) children.push({ group: 'children:', - html_url: url.href + html_url: getUrlStringForChildrenLine(currentLine, issue) }) } catch { // invalid URL or child issue identifier, exit and return what we have From 7a80467bbb5fc3ecaa5e8dcfba2802e2cc396c1f Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 10 Feb 2023 19:18:44 -0800 Subject: [PATCH 5/7] docs: update User Guide with tasklist instructions --- User Guide.md | 60 ++++++++++++++++++++++++++++++++++++++++++++++++--- lib/parser.ts | 2 +- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/User Guide.md b/User Guide.md index 4c64e573..07a4fe08 100644 --- a/User Guide.md +++ b/User Guide.md @@ -40,6 +40,7 @@ The fundamental unit of a roadmap is a project milestone. In the context of road - Roadmaps are represented by a single root node, or GitHub issue, which contains links to milestones contained within that roadmap. - The roadmap root node and child milestones can be in any public repository as long as the issues satisfy the requirements outlined in this document. - This means that you can link to existing GitHub issues as child milestones. +- Errors will be logged and displayed for the user in starmap.site ### Milestone Encodings @@ -93,18 +94,38 @@ can expect to get from this milestone. #### Children - A milestone may have child milestones. -- Child milestones are simply full URL links to other GitHub milestone issues. -- Child milestones can exist in any public Github repository. +- Child milestones are simply GitHub issue identifiers (#, /#, or full URLs) to other GitHub milestone issues. +- Child milestones can exist in any public GitHub repository. - It is expected that child milestone issues are themselves properly encoded milestones; otherwise they will be ignored by Starmap. - Within a parent issue, child milestones are encoded as follows (raw Markdown): +##### Tasklist syntax + +Tasklists allow for "taskifying" of strings, and we have no way to link a random string to a GitHub issue. You must convert any [tasks to issues](https://docs.github.com/en/issues/tracking-your-work-with-issues/about-tasklists#converting-draft-issues-to-issues-in-a-tasklist) for them to show up as a child milestone. + +We will do our best to support the expected syntax of GitHub's tasklist functionality. + +See https://docs.github.com/en/issues/tracking-your-work-with-issues/about-tasklists#creating-tasklists and https://github.com/pln-planning-tools/Starmap/issues/245 for more details. + +``` +```[tasklist] +### Tasks +- [ ] https://github.com/pln-roadmap/Roadmap-Vizualizer/issues/10 +- [ ] https://github.com/pln-roadmap/Roadmap-Vizualizer/issues/9 +- [ ] https://github.com/pln-roadmap/Roadmap-Vizualizer/issues/8 +\``` +``` + +##### "Children:" syntax + +This syntax is deprecated. Please see https://github.com/pln-planning-tools/Starmap/issues/245 for more details. + ``` Children: - https://github.com/pln-roadmap/Roadmap-Vizualizer/issues/10 - https://github.com/pln-roadmap/Roadmap-Vizualizer/issues/9 - https://github.com/pln-roadmap/Roadmap-Vizualizer/issues/8 ``` -- Errors will be logged and displayed for the user in starmap.site ### Progress Indicators @@ -127,6 +148,39 @@ Children: ### Templates #### Root Node Issue + +##### Using GitHub Tasklists + +``` +Title: [Team/Project Name] [Duration] Roadmap + +Description (optional): +The goal of this roadmap is to outline the key milestones and deliverables for our team/project over the next [Duration]. + +```[tasklist] +### Any descriptor or other text +- [ ] #123 +- [ ] org/repo#123 +- [ ] some non-link description +- [ ] https://github.com/org/repo/issue/987 + +### Any text +- [ ] #456 +- [ ] org/repo#567 +- [ ] https://github.com/other-org/other-repo/issue/987 + +\``` + +Note: This roadmap is subject to change as priorities and circumstances evolve. + +Starmap Link: [Starmap Link] + +``` + +##### Using "Children:" + +**NOTE:** The children: section is deprecated. Please see https://github.com/pln-planning-tools/Starmap/issues/245 for more details + ``` Title: [Team/Project Name] [Duration] Roadmap diff --git a/lib/parser.ts b/lib/parser.ts index af3f133b..530b6316 100644 --- a/lib/parser.ts +++ b/lib/parser.ts @@ -71,7 +71,7 @@ function getChildrenFromTaskList(issue: Pick): ParserGetChildrenResponse[] { +function getChildrenNew(issue: Pick): ParserGetChildrenResponse[] { try { return getChildrenFromTaskList(issue); From 63b87fafabb81ecbc1fd21be36c9f919d4440063 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Thu, 16 Feb 2023 13:47:31 -0800 Subject: [PATCH 6/7] Update lib/parser.ts Co-authored-by: Nishant Arora <1895906+whizzzkid@users.noreply.github.com> --- lib/parser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/parser.ts b/lib/parser.ts index 530b6316..5b1e645e 100644 --- a/lib/parser.ts +++ b/lib/parser.ts @@ -34,7 +34,7 @@ function getSectionLines(text: string, sectionHeader: string) { if (sectionIndex === -1) { return []; } - const lines = text.substring(sectionIndex).split(/\r\n|\r|\n/).slice(1); + const lines = text.substring(sectionIndex).split(/[\r\n]+/).slice(1); return lines; } From 560625029282837971ffe2c9761ec1bcf87109b5 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Thu, 16 Feb 2023 14:24:08 -0800 Subject: [PATCH 7/7] chore: address PR comments --- lib/parser.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/parser.ts b/lib/parser.ts index 5b1e645e..efdfc423 100644 --- a/lib/parser.ts +++ b/lib/parser.ts @@ -38,6 +38,9 @@ function getSectionLines(text: string, sectionHeader: string) { return lines; } +const splitAndGetLastItem = (line: string) => line.trim().split(' ').slice(-1)[0] +const ensureTaskListChild = (line: string) => line.trim().indexOf('-') === 0 + function getUrlStringForChildrenLine(line: string, issue: Pick) { if (/^#\d+$/.test(line)) { const { owner, repo } = paramsFromUrl(issue.html_url) @@ -53,7 +56,11 @@ function getUrlStringForChildrenLine(line: string, issue: Pick): ParserGetChildrenResponse[] { // tasklists require the checkbox style format to recognize children - const lines = getSectionLines(issue.body, '```[tasklist]').filter((line) => line.trim().indexOf('-') === 0).map((line) => line.trim().split(' ').slice(-1)[0]).filter(Boolean); + const lines = getSectionLines(issue.body, '```[tasklist]') + .filter(ensureTaskListChild) + .map(splitAndGetLastItem) + .filter(Boolean); + if (lines.length === 0) { throw new Error('Section missing or has no children') } @@ -83,6 +90,12 @@ function getChildrenNew(issue: Pick): Pars if (lines.length === 0) { throw new Error('Section missing or has no children') } + + // guard against HTML tags (covers cases where this method is called with issue.body_html instead of issue.body_text) + if (lines.some((line) => line.startsWith('<'))) { + throw new Error('HTML tags found in body_text'); + } + const children: ParserGetChildrenResponse[] = [] for (let i = 0; i < lines.length; i++) { @@ -96,10 +109,6 @@ function getChildrenNew(issue: Pick): Pars break } } - // guard against HTML tags (covers cases where this method is called with issue.body_html instead of issue.body_text) - if (currentLine.startsWith('<')) { - throw new Error('HTML tags found in body_text'); - } try { children.push({