From 28cec18b46897a4b31b6003a92d5dcda39c9078f Mon Sep 17 00:00:00 2001 From: Boris Lykah Date: Sun, 20 Feb 2022 17:56:44 -0700 Subject: [PATCH] Collect and report number of changes --- CHANGELOG.md | 3 + README.md | 45 +++++++-- __tests__/filter.test.ts | 12 +-- __tests__/git.test.ts | 15 ++- action.yml | 14 ++- dist/index.js | 205 ++++++++++++++++++++++++++------------- package-lock.json | 6 +- src/file.ts | 10 +- src/filter.ts | 13 ++- src/git.ts | 128 +++++++++++++----------- src/main.ts | 90 +++++++++++++---- 11 files changed, 373 insertions(+), 168 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f3a7af3..8cd20f46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +# v3.0.3 +- Added stat feature which computes diffs in subtrees + ## v3.0.2 - [Add config parameter for predicate quantifier](https://github.com/dorny/paths-filter/pull/224) diff --git a/README.md b/README.md index b5e0f4cf..d2c6b3fe 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Paths Changes Filter +# Paths Changes Filter And Diff Stat [GitHub Action](https://github.com/features/actions) that enables conditional execution of workflow steps and jobs, based on the files modified by pull request, on a feature branch, or by the recently pushed commits. @@ -73,6 +73,7 @@ For more scenarios see [examples](#examples) section. ## What's New - New major release `v3` after update to Node 20 [Breaking change] +- Add `stat` parameter that enables output of the file changes statistics per filter. - Add `ref` input parameter - Add `list-files: csv` format - Configure matrix job to run for each folder with changes using `changes` output @@ -154,14 +155,14 @@ For more information, see [CHANGELOG](https://github.com/dorny/paths-filter/blob # Default: ${{ github.token }} token: '' - # Optional parameter to override the default behavior of file matching algorithm. + # Optional parameter to override the default behavior of file matching algorithm. # By default files that match at least one pattern defined by the filters will be included. # This parameter allows to override the "at least one pattern" behavior to make it so that - # all of the patterns have to match or otherwise the file is excluded. - # An example scenario where this is useful if you would like to match all - # .ts files in a sub-directory but not .md files. - # The filters below will match markdown files despite the exclusion syntax UNLESS - # you specify 'every' as the predicate-quantifier parameter. When you do that, + # all of the patterns have to match or otherwise the file is excluded. + # An example scenario where this is useful if you would like to match all + # .ts files in a sub-directory but not .md files. + # The filters below will match markdown files despite the exclusion syntax UNLESS + # you specify 'every' as the predicate-quantifier parameter. When you do that, # it will only match the .ts files in the subdirectory as expected. # # backend: @@ -179,6 +180,7 @@ For more information, see [CHANGELOG](https://github.com/dorny/paths-filter/blob - For each filter, it sets an output variable with the name `${FILTER_NAME}_count` to the count of matching files. - If enabled, for each filter it sets an output variable with the name `${FILTER_NAME}_files`. It will contain a list of all files matching the filter. - `changes` - JSON array with names of all filters matching any of the changed files. +- If `stat` input is set to an output format, the output variable `stat` contains JSON or CSV value with the change statistics for each filter. ## Examples @@ -558,10 +560,37 @@ jobs: +
+ Passing number of added lines from a filter to another action + +```yaml +- uses: dorny/paths-filter@v2 + id: filter + with: + # Enable listing of diff stat matching each filter. + # Paths to files will be available in `stat` output variable. + # Stat will be formatted as JSON object + stat: json + + # In this example all changed files are passed to the following action to do + # some custom processing. + filters: | + changed: + - '**' +- name: Lint Markdown + uses: johndoe/some-action@v1 + # Run action only if the change is large enough. + if: ${{fromJson(steps.filter.outputs.stat).changed.additionCount > 1000}} + with: + files: ${{ steps.filter.outputs.changed_files }} +``` + +
+ ## See also - [test-reporter](https://github.com/dorny/test-reporter) - Displays test results from popular testing frameworks directly in GitHub ## License -The scripts and documentation in this project are released under the [MIT License](https://github.com/dorny/paths-filter/blob/master/LICENSE) +The scripts and documentation in this project are released under the [MIT License](https://github.com/lykahb/paths-filter/blob/master/LICENSE) diff --git a/__tests__/filter.test.ts b/__tests__/filter.test.ts index 7d7da947..7d764676 100644 --- a/__tests__/filter.test.ts +++ b/__tests__/filter.test.ts @@ -181,7 +181,7 @@ describe('matching specific change status', () => { - added: "**/*" ` let filter = new Filter(yaml) - const files = [{status: ChangeStatus.Added, filename: 'file.js'}] + const files = [{status: ChangeStatus.Added, filename: 'file.js', additions: 1, deletions: 0}] const match = filter.match(files) expect(match.add).toEqual(files) }) @@ -192,7 +192,7 @@ describe('matching specific change status', () => { - added|modified: "**/*" ` let filter = new Filter(yaml) - const files = [{status: ChangeStatus.Modified, filename: 'file.js'}] + const files = [{status: ChangeStatus.Modified, filename: 'file.js', additions: 1, deletions: 1}] const match = filter.match(files) expect(match.addOrModify).toEqual(files) }) @@ -214,12 +214,6 @@ describe('matching specific change status', () => { function modified(paths: string[]): File[] { return paths.map(filename => { - return {filename, status: ChangeStatus.Modified} - }) -} - -function renamed(paths: string[]): File[] { - return paths.map(filename => { - return {filename, status: ChangeStatus.Renamed} + return {filename, status: ChangeStatus.Modified, additions: 1, deletions: 1} }) } diff --git a/__tests__/git.test.ts b/__tests__/git.test.ts index 9645221e..5597f040 100644 --- a/__tests__/git.test.ts +++ b/__tests__/git.test.ts @@ -2,8 +2,8 @@ import * as git from '../src/git' import {ChangeStatus} from '../src/file' describe('parsing output of the git diff command', () => { - test('parseGitDiffOutput returns files with correct change status', async () => { - const files = git.parseGitDiffOutput( + test('parseGitDiffNameStatusOutput returns files with correct change status', async () => { + const files = git.parseGitDiffNameStatusOutput( 'A\u0000LICENSE\u0000' + 'M\u0000src/index.ts\u0000' + 'D\u0000src/main.ts\u0000' ) expect(files.length).toBe(3) @@ -14,6 +14,17 @@ describe('parsing output of the git diff command', () => { expect(files[2].filename).toBe('src/main.ts') expect(files[2].status).toBe(ChangeStatus.Deleted) }) + + test('parseGitDiffNumstatOutput returns files with correct change status', async () => { + const files = git.parseGitDiffNumstatOutput('4\t2\tLICENSE\u0000' + '5\t0\tsrc/index.ts\u0000') + expect(files.length).toBe(2) + expect(files[0].filename).toBe('LICENSE') + expect(files[0].additions).toBe(4) + expect(files[0].deletions).toBe(2) + expect(files[1].filename).toBe('src/index.ts') + expect(files[1].additions).toBe(5) + expect(files[1].deletions).toBe(0) + }) }) describe('git utility function tests (those not invoking git)', () => { diff --git a/action.yml b/action.yml index e7d24f5c..341f3137 100644 --- a/action.yml +++ b/action.yml @@ -1,6 +1,6 @@ -name: 'Paths Changes Filter' +name: 'Paths Changes Filter And Diff Stat' description: 'Execute your workflow steps only if relevant files are modified.' -author: 'Michal Dorner ' +author: 'Michal Dorner , Boris Lykah' inputs: token: description: 'GitHub Access Token' @@ -36,6 +36,16 @@ inputs: Backslash escapes every potentially unsafe character. required: false default: none + stat: + description: | + Enables listing of that enables output of the file change statistics per filter, similar to `git diff --shortstat`. + If some changes do not match any filter, the output includes an additional entry with the filter name 'other'. + 'none' - Disables listing of stats (default). + 'csv' - Coma separated list that has name of filter, count of additions, count of deletions, count of changed files. + If needed it uses double quotes to wrap name of filter with unsafe characters. For example, `"some filter",12,7,2`. + 'json' - Serialized as JSON object where the filter names are keys. For example, `{"some filter": {"additionCount": 12, "deletionCount": 7, "fileCount": 2}}` + required: false + default: none initial-fetch-depth: description: | How many commits are initially fetched from base branch. diff --git a/dist/index.js b/dist/index.js index cc7d7d43..9100fa7c 100644 --- a/dist/index.js +++ b/dist/index.js @@ -123,6 +123,11 @@ class Filter { for (const [key, patterns] of Object.entries(this.rules)) { result[key] = files.filter(file => this.isMatch(file, patterns)); } + if (!this.rules.hasOwnProperty('other')) { + const matchingFilenamesList = Object.values(result).flatMap(filteredFiles => filteredFiles.map(file => file.filename)); + const matchingFilenamesSet = new Set(matchingFilenamesList); + result.other = files.filter(file => !matchingFilenamesSet.has(file.filename)); + } return result; } isMatch(file, patterns) { @@ -204,55 +209,39 @@ var __importStar = (this && this.__importStar) || function (mod) { return result; }; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.isGitSha = exports.getShortName = exports.getCurrentRef = exports.listAllFilesAsAdded = exports.parseGitDiffOutput = exports.getChangesSinceMergeBase = exports.getChangesOnHead = exports.getChanges = exports.getChangesInLastCommit = exports.HEAD = exports.NULL_SHA = void 0; +exports.isGitSha = exports.getShortName = exports.getCurrentRef = exports.listAllFilesAsAdded = exports.parseGitDiffNumstatOutput = exports.getGitDiffStatusNumstat = exports.parseGitDiffNameStatusOutput = exports.getChangesSinceMergeBase = exports.getChangesOnHead = exports.getChanges = exports.getChangesInLastCommit = exports.HEAD = exports.NULL_SHA = void 0; const exec_1 = __nccwpck_require__(1514); const core = __importStar(__nccwpck_require__(2186)); const file_1 = __nccwpck_require__(4014); exports.NULL_SHA = '0000000000000000000000000000000000000000'; exports.HEAD = 'HEAD'; async function getChangesInLastCommit() { - core.startGroup(`Change detection in last commit`); - let output = ''; - try { - output = (await (0, exec_1.getExecOutput)('git', ['log', '--format=', '--no-renames', '--name-status', '-z', '-n', '1'])).stdout; - } - finally { - fixStdOutNullTermination(); - core.endGroup(); - } - return parseGitDiffOutput(output); + return core.group(`Change detection in last commit`, async () => { + try { + // Calling git log on the last commit works when only the last commit may be checked out. Calling git diff HEAD^..HEAD needs two commits. + const statusOutput = (await (0, exec_1.getExecOutput)('git', ['log', '--format=', '--no-renames', '--name-status', '-z', '-n', '1'])).stdout; + const numstatOutput = (await (0, exec_1.getExecOutput)('git', ['log', '--format=', '--no-renames', '--numstat', '-z', '-n', '1'])).stdout; + const statusFiles = parseGitDiffNameStatusOutput(statusOutput); + const numstatFiles = parseGitDiffNumstatOutput(numstatOutput); + return mergeStatusNumstat(statusFiles, numstatFiles); + } + finally { + fixStdOutNullTermination(); + } + }); } exports.getChangesInLastCommit = getChangesInLastCommit; async function getChanges(base, head) { const baseRef = await ensureRefAvailable(base); const headRef = await ensureRefAvailable(head); // Get differences between ref and HEAD - core.startGroup(`Change detection ${base}..${head}`); - let output = ''; - try { - // Two dots '..' change detection - directly compares two versions - output = (await (0, exec_1.getExecOutput)('git', ['diff', '--no-renames', '--name-status', '-z', `${baseRef}..${headRef}`])) - .stdout; - } - finally { - fixStdOutNullTermination(); - core.endGroup(); - } - return parseGitDiffOutput(output); + // Two dots '..' change detection - directly compares two versions + return core.group(`Change detection ${base}..${head}`, () => getGitDiffStatusNumstat(`${baseRef}..${headRef}`)); } exports.getChanges = getChanges; async function getChangesOnHead() { // Get current changes - both staged and unstaged - core.startGroup(`Change detection on HEAD`); - let output = ''; - try { - output = (await (0, exec_1.getExecOutput)('git', ['diff', '--no-renames', '--name-status', '-z', 'HEAD'])).stdout; - } - finally { - fixStdOutNullTermination(); - core.endGroup(); - } - return parseGitDiffOutput(output); + return core.group(`Change detection on HEAD`, () => getGitDiffStatusNumstat(`HEAD`)); } exports.getChangesOnHead = getChangesOnHead; async function getChangesSinceMergeBase(base, head, initialFetchDepth) { @@ -317,19 +306,30 @@ async function getChangesSinceMergeBase(base, head, initialFetchDepth) { diffArg = `${baseRef}..${headRef}`; } // Get changes introduced on ref compared to base - core.startGroup(`Change detection ${diffArg}`); + return getGitDiffStatusNumstat(diffArg); +} +exports.getChangesSinceMergeBase = getChangesSinceMergeBase; +async function gitDiffNameStatus(diffArg) { let output = ''; try { output = (await (0, exec_1.getExecOutput)('git', ['diff', '--no-renames', '--name-status', '-z', diffArg])).stdout; } finally { fixStdOutNullTermination(); - core.endGroup(); } - return parseGitDiffOutput(output); + return output; } -exports.getChangesSinceMergeBase = getChangesSinceMergeBase; -function parseGitDiffOutput(output) { +async function gitDiffNumstat(diffArg) { + let output = ''; + try { + output = (await (0, exec_1.getExecOutput)('git', ['diff', '--no-renames', '--numstat', '-z', diffArg])).stdout; + } + finally { + fixStdOutNullTermination(); + } + return output; +} +function parseGitDiffNameStatusOutput(output) { const tokens = output.split('\u0000').filter(s => s.length > 0); const files = []; for (let i = 0; i + 1 < tokens.length; i += 2) { @@ -340,24 +340,44 @@ function parseGitDiffOutput(output) { } return files; } -exports.parseGitDiffOutput = parseGitDiffOutput; +exports.parseGitDiffNameStatusOutput = parseGitDiffNameStatusOutput; +function mergeStatusNumstat(statusEntries, numstatEntries) { + const statusMap = {}; + statusEntries.forEach(f => (statusMap[f.filename] = f)); + return numstatEntries.map(f => { + const status = statusMap[f.filename]; + if (!status) { + throw new Error(`Cannot find the status entry for file: ${f.filename}`); + } + return { ...f, status: status.status }; + }); +} +async function getGitDiffStatusNumstat(diffArg) { + const statusFiles = await gitDiffNameStatus(diffArg).then(parseGitDiffNameStatusOutput); + const numstatFiles = await gitDiffNumstat(diffArg).then(parseGitDiffNumstatOutput); + return mergeStatusNumstat(statusFiles, numstatFiles); +} +exports.getGitDiffStatusNumstat = getGitDiffStatusNumstat; +function parseGitDiffNumstatOutput(output) { + const rows = output.split('\u0000').filter(s => s.length > 0); + return rows.map(row => { + const tokens = row.split('\t'); + // For the binary files set the numbers to zero. This matches the response of Github API. + const additions = tokens[0] == '-' ? 0 : Number.parseInt(tokens[0]); + const deletions = tokens[1] == '-' ? 0 : Number.parseInt(tokens[1]); + return { + filename: tokens[2], + additions, + deletions + }; + }); +} +exports.parseGitDiffNumstatOutput = parseGitDiffNumstatOutput; async function listAllFilesAsAdded() { - core.startGroup('Listing all files tracked by git'); - let output = ''; - try { - output = (await (0, exec_1.getExecOutput)('git', ['ls-files', '-z'])).stdout; - } - finally { - fixStdOutNullTermination(); - core.endGroup(); - } - return output - .split('\u0000') - .filter(s => s.length > 0) - .map(path => ({ - status: file_1.ChangeStatus.Added, - filename: path - })); + return core.group(`Listing all files tracked by git`, async () => { + const emptyTreeHash = (await (0, exec_1.getExecOutput)('git', ['hash-object', '-t', 'tree', '/dev/null'])).stdout; + return getGitDiffStatusNumstat(emptyTreeHash); + }); } exports.listAllFilesAsAdded = listAllFilesAsAdded; async function getCurrentRef() { @@ -572,11 +592,16 @@ async function run() { const base = core.getInput('base', { required: false }); const filtersInput = core.getInput('filters', { required: true }); const filtersYaml = isPathInput(filtersInput) ? getConfigFileContent(filtersInput) : filtersInput; - const listFiles = core.getInput('list-files', { required: false }).toLowerCase() || 'none'; + const listFilesFormat = core.getInput('list-files', { required: false }).toLowerCase() || 'none'; + const statFormat = core.getInput('stat', { required: false }).toLowerCase() || 'none'; const initialFetchDepth = parseInt(core.getInput('initial-fetch-depth', { required: false })) || 10; const predicateQuantifier = core.getInput('predicate-quantifier', { required: false }) || filter_1.PredicateQuantifier.SOME; - if (!isExportFormat(listFiles)) { - core.setFailed(`Input parameter 'list-files' is set to invalid value '${listFiles}'`); + if (!isFilesExportFormat(listFilesFormat)) { + core.setFailed(`Input parameter 'list-files' is set to invalid value '${listFilesFormat}'`); + return; + } + if (!isStatExportFormat(statFormat)) { + core.setFailed(`Input parameter 'stat' is set to invalid value '${statFormat}'`); return; } if (!(0, filter_1.isPredicateQuantifier)(predicateQuantifier)) { @@ -589,7 +614,7 @@ async function run() { const files = await getChangedFiles(token, base, ref, initialFetchDepth); core.info(`Detected ${files.length} changed files`); const results = filter.match(files); - exportResults(results, listFiles); + exportResults(results, listFilesFormat, statFormat); } catch (error) { core.setFailed(getErrorMessage(error)); @@ -718,12 +743,16 @@ async function getChangedFilesFromApi(token, pullRequest) { if (row.status === file_1.ChangeStatus.Renamed) { files.push({ filename: row.filename, - status: file_1.ChangeStatus.Added + status: file_1.ChangeStatus.Added, + additions: row.additions, + deletions: row.deletions }); files.push({ // 'previous_filename' for some unknown reason isn't in the type definition or documentation filename: row.previous_filename, - status: file_1.ChangeStatus.Deleted + status: file_1.ChangeStatus.Deleted, + additions: row.additions, + deletions: row.deletions }); } else { @@ -731,7 +760,9 @@ async function getChangedFilesFromApi(token, pullRequest) { const status = row.status === 'removed' ? file_1.ChangeStatus.Deleted : row.status; files.push({ filename: row.filename, - status + status, + additions: row.additions, + deletions: row.deletions }); } } @@ -742,12 +773,13 @@ async function getChangedFilesFromApi(token, pullRequest) { core.endGroup(); } } -function exportResults(results, format) { +function exportResults(results, filesFormat, statFormat) { core.info('Results:'); const changes = []; + const changeStats = {}; for (const [key, files] of Object.entries(results)) { - const value = files.length > 0; - core.startGroup(`Filter ${key} = ${value}`); + const hasMatchingFiles = files.length > 0; + core.startGroup(`Filter ${key} = ${hasMatchingFiles}`); if (files.length > 0) { changes.push(key); core.info('Matching files:'); @@ -758,12 +790,22 @@ function exportResults(results, format) { else { core.info('Matching files: none'); } - core.setOutput(key, value); + core.setOutput(key, hasMatchingFiles); core.setOutput(`${key}_count`, files.length); - if (format !== 'none') { - const filesValue = serializeExport(files, format); + if (filesFormat !== 'none') { + const filesValue = serializeExportChangedFiles(files, filesFormat); core.setOutput(`${key}_files`, filesValue); } + const additionCount = files.reduce((sum, f) => sum + f.additions, 0); + const deletionCount = files.reduce((sum, f) => sum + f.deletions, 0); + core.setOutput(`${key}_addition_count`, additionCount); + core.setOutput(`${key}_deletion_count`, deletionCount); + core.setOutput(`${key}_change_count`, additionCount + deletionCount); + changeStats[key] = { + additionCount, + deletionCount, + fileCount: files.length + }; core.endGroup(); } if (results['changes'] === undefined) { @@ -774,8 +816,12 @@ function exportResults(results, format) { else { core.info('Cannot set changes output variable - name already used by filter output'); } + if (statFormat !== 'none') { + const statValue = serializeExportStat(changeStats, statFormat); + core.setOutput(`stat`, statValue); + } } -function serializeExport(files, format) { +function serializeExportChangedFiles(files, format) { const fileNames = files.map(file => file.filename); switch (format) { case 'csv': @@ -790,7 +836,20 @@ function serializeExport(files, format) { return ''; } } -function isExportFormat(value) { +function serializeExportStat(stat, format) { + switch (format) { + case 'csv': + return Object.keys(stat) + .sort() + .map(k => [(0, csv_escape_1.csvEscape)(k), stat[k].additionCount, stat[k].deletionCount, stat[k].fileCount].join(',')) + .join('\n'); + case 'json': + return JSON.stringify(stat); + default: + return ''; + } +} +function isFilesExportFormat(value) { return ['none', 'csv', 'shell', 'json', 'escape'].includes(value); } function getErrorMessage(error) { @@ -798,6 +857,9 @@ function getErrorMessage(error) { return error.message; return String(error); } +function isStatExportFormat(value) { + return ['none', 'csv', 'json'].includes(value); +} run(); @@ -26291,6 +26353,9 @@ function httpRedirectFetch (fetchParams, response) { // https://fetch.spec.whatwg.org/#cors-non-wildcard-request-header-name request.headersList.delete('authorization') + // https://fetch.spec.whatwg.org/#authentication-entries + request.headersList.delete('proxy-authorization', true) + // "Cookie" and "Host" are forbidden request-headers, which undici doesn't implement. request.headersList.delete('cookie') request.headersList.delete('host') diff --git a/package-lock.json b/package-lock.json index d1a475f3..b91c6d53 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6813,9 +6813,9 @@ } }, "node_modules/undici": { - "version": "5.28.2", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.2.tgz", - "integrity": "sha512-wh1pHJHnUeQV5Xa8/kyQhO7WFa8M34l026L5P/+2TYiakvGy5Rdc8jWZVyG7ieht/0WgJLEd3kcU5gKx+6GC8w==", + "version": "5.28.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.3.tgz", + "integrity": "sha512-3ItfzbrhDlINjaP0duwnNsKpDQk3acHI3gVJ1z4fmwMK31k5G9OVIAMLSIaP6w4FaGkaAkN6zaQO9LUvZ1t7VA==", "dependencies": { "@fastify/busboy": "^2.0.0" }, diff --git a/src/file.ts b/src/file.ts index d8125a7a..b5d3ea27 100644 --- a/src/file.ts +++ b/src/file.ts @@ -1,8 +1,16 @@ -export interface File { +export interface FileStatus { filename: string status: ChangeStatus } +export interface FileNumstat { + filename: string + additions: number + deletions: number +} + +export type File = FileStatus & FileNumstat + export enum ChangeStatus { Added = 'added', Copied = 'copied', diff --git a/src/filter.ts b/src/filter.ts index 1947ef88..2fec30d0 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -1,6 +1,6 @@ import * as jsyaml from 'js-yaml' import picomatch from 'picomatch' -import {File, ChangeStatus} from './file' +import {File, ChangeStatus, FileStatus} from './file' // Type definition of object we expect to load from YAML interface FilterYaml { @@ -100,10 +100,19 @@ export class Filter { for (const [key, patterns] of Object.entries(this.rules)) { result[key] = files.filter(file => this.isMatch(file, patterns)) } + + if (!this.rules.hasOwnProperty('other')) { + const matchingFilenamesList = Object.values(result).flatMap(filteredFiles => + filteredFiles.map(file => file.filename) + ) + const matchingFilenamesSet = new Set(matchingFilenamesList) + result.other = files.filter(file => !matchingFilenamesSet.has(file.filename)) + } + return result } - private isMatch(file: File, patterns: FilterRuleItem[]): boolean { + private isMatch(file: FileStatus, patterns: FilterRuleItem[]): boolean { const aPredicate = (rule: Readonly): boolean => { return (rule.status === undefined || rule.status.includes(file.status)) && rule.isMatch(file.filename) } diff --git a/src/git.ts b/src/git.ts index a4132362..d883c97c 100644 --- a/src/git.ts +++ b/src/git.ts @@ -1,21 +1,27 @@ import {getExecOutput} from '@actions/exec' import * as core from '@actions/core' -import {File, ChangeStatus} from './file' +import {File, ChangeStatus, FileNumstat, FileStatus} from './file' export const NULL_SHA = '0000000000000000000000000000000000000000' export const HEAD = 'HEAD' export async function getChangesInLastCommit(): Promise { - core.startGroup(`Change detection in last commit`) - let output = '' - try { - output = (await getExecOutput('git', ['log', '--format=', '--no-renames', '--name-status', '-z', '-n', '1'])).stdout - } finally { - fixStdOutNullTermination() - core.endGroup() - } - - return parseGitDiffOutput(output) + return core.group(`Change detection in last commit`, async () => { + try { + // Calling git log on the last commit works when only the last commit may be checked out. Calling git diff HEAD^..HEAD needs two commits. + const statusOutput = ( + await getExecOutput('git', ['log', '--format=', '--no-renames', '--name-status', '-z', '-n', '1']) + ).stdout + const numstatOutput = ( + await getExecOutput('git', ['log', '--format=', '--no-renames', '--numstat', '-z', '-n', '1']) + ).stdout + const statusFiles = parseGitDiffNameStatusOutput(statusOutput) + const numstatFiles = parseGitDiffNumstatOutput(numstatOutput) + return mergeStatusNumstat(statusFiles, numstatFiles) + } finally { + fixStdOutNullTermination() + } + }) } export async function getChanges(base: string, head: string): Promise { @@ -23,32 +29,13 @@ export async function getChanges(base: string, head: string): Promise { const headRef = await ensureRefAvailable(head) // Get differences between ref and HEAD - core.startGroup(`Change detection ${base}..${head}`) - let output = '' - try { - // Two dots '..' change detection - directly compares two versions - output = (await getExecOutput('git', ['diff', '--no-renames', '--name-status', '-z', `${baseRef}..${headRef}`])) - .stdout - } finally { - fixStdOutNullTermination() - core.endGroup() - } - - return parseGitDiffOutput(output) + // Two dots '..' change detection - directly compares two versions + return core.group(`Change detection ${base}..${head}`, () => getGitDiffStatusNumstat(`${baseRef}..${headRef}`)) } export async function getChangesOnHead(): Promise { // Get current changes - both staged and unstaged - core.startGroup(`Change detection on HEAD`) - let output = '' - try { - output = (await getExecOutput('git', ['diff', '--no-renames', '--name-status', '-z', 'HEAD'])).stdout - } finally { - fixStdOutNullTermination() - core.endGroup() - } - - return parseGitDiffOutput(output) + return core.group(`Change detection on HEAD`, () => getGitDiffStatusNumstat(`HEAD`)) } export async function getChangesSinceMergeBase(base: string, head: string, initialFetchDepth: number): Promise { @@ -120,21 +107,32 @@ export async function getChangesSinceMergeBase(base: string, head: string, initi } // Get changes introduced on ref compared to base - core.startGroup(`Change detection ${diffArg}`) + return getGitDiffStatusNumstat(diffArg) +} + +async function gitDiffNameStatus(diffArg: string): Promise { let output = '' try { output = (await getExecOutput('git', ['diff', '--no-renames', '--name-status', '-z', diffArg])).stdout } finally { fixStdOutNullTermination() - core.endGroup() } + return output +} - return parseGitDiffOutput(output) +async function gitDiffNumstat(diffArg: string): Promise { + let output = '' + try { + output = (await getExecOutput('git', ['diff', '--no-renames', '--numstat', '-z', diffArg])).stdout + } finally { + fixStdOutNullTermination() + } + return output } -export function parseGitDiffOutput(output: string): File[] { +export function parseGitDiffNameStatusOutput(output: string): FileStatus[] { const tokens = output.split('\u0000').filter(s => s.length > 0) - const files: File[] = [] + const files: FileStatus[] = [] for (let i = 0; i + 1 < tokens.length; i += 2) { files.push({ status: statusMap[tokens[i]], @@ -144,23 +142,45 @@ export function parseGitDiffOutput(output: string): File[] { return files } -export async function listAllFilesAsAdded(): Promise { - core.startGroup('Listing all files tracked by git') - let output = '' - try { - output = (await getExecOutput('git', ['ls-files', '-z'])).stdout - } finally { - fixStdOutNullTermination() - core.endGroup() - } +function mergeStatusNumstat(statusEntries: FileStatus[], numstatEntries: FileNumstat[]): File[] { + const statusMap: {[key: string]: FileStatus} = {} + statusEntries.forEach(f => (statusMap[f.filename] = f)) - return output - .split('\u0000') - .filter(s => s.length > 0) - .map(path => ({ - status: ChangeStatus.Added, - filename: path - })) + return numstatEntries.map(f => { + const status = statusMap[f.filename] + if (!status) { + throw new Error(`Cannot find the status entry for file: ${f.filename}`) + } + return {...f, status: status.status} + }) +} + +export async function getGitDiffStatusNumstat(diffArg: string) { + const statusFiles = await gitDiffNameStatus(diffArg).then(parseGitDiffNameStatusOutput) + const numstatFiles = await gitDiffNumstat(diffArg).then(parseGitDiffNumstatOutput) + return mergeStatusNumstat(statusFiles, numstatFiles) +} + +export function parseGitDiffNumstatOutput(output: string): FileNumstat[] { + const rows = output.split('\u0000').filter(s => s.length > 0) + return rows.map(row => { + const tokens = row.split('\t') + // For the binary files set the numbers to zero. This matches the response of Github API. + const additions = tokens[0] == '-' ? 0 : Number.parseInt(tokens[0]) + const deletions = tokens[1] == '-' ? 0 : Number.parseInt(tokens[1]) + return { + filename: tokens[2], + additions, + deletions + } + }) +} + +export async function listAllFilesAsAdded(): Promise { + return core.group(`Listing all files tracked by git`, async () => { + const emptyTreeHash = (await getExecOutput('git', ['hash-object', '-t', 'tree', '/dev/null'])).stdout + return getGitDiffStatusNumstat(emptyTreeHash) + }) } export async function getCurrentRef(): Promise { diff --git a/src/main.ts b/src/main.ts index 8320287c..edeb56fb 100644 --- a/src/main.ts +++ b/src/main.ts @@ -17,7 +17,8 @@ import * as git from './git' import {backslashEscape, shellEscape} from './list-format/shell-escape' import {csvEscape} from './list-format/csv-escape' -type ExportFormat = 'none' | 'csv' | 'json' | 'shell' | 'escape' +type FilesExportFormat = 'none' | 'csv' | 'json' | 'shell' | 'escape' +type StatExportFormat = 'none' | 'csv' | 'json' async function run(): Promise { try { @@ -31,12 +32,18 @@ async function run(): Promise { const base = core.getInput('base', {required: false}) const filtersInput = core.getInput('filters', {required: true}) const filtersYaml = isPathInput(filtersInput) ? getConfigFileContent(filtersInput) : filtersInput - const listFiles = core.getInput('list-files', {required: false}).toLowerCase() || 'none' + const listFilesFormat = core.getInput('list-files', {required: false}).toLowerCase() || 'none' + const statFormat = core.getInput('stat', {required: false}).toLowerCase() || 'none' const initialFetchDepth = parseInt(core.getInput('initial-fetch-depth', {required: false})) || 10 const predicateQuantifier = core.getInput('predicate-quantifier', {required: false}) || PredicateQuantifier.SOME - if (!isExportFormat(listFiles)) { - core.setFailed(`Input parameter 'list-files' is set to invalid value '${listFiles}'`) + if (!isFilesExportFormat(listFilesFormat)) { + core.setFailed(`Input parameter 'list-files' is set to invalid value '${listFilesFormat}'`) + return + } + + if (!isStatExportFormat(statFormat)) { + core.setFailed(`Input parameter 'stat' is set to invalid value '${statFormat}'`) return } @@ -52,7 +59,7 @@ async function run(): Promise { const files = await getChangedFiles(token, base, ref, initialFetchDepth) core.info(`Detected ${files.length} changed files`) const results = filter.match(files) - exportResults(results, listFiles) + exportResults(results, listFilesFormat, statFormat) } catch (error) { core.setFailed(getErrorMessage(error)) } @@ -204,19 +211,25 @@ async function getChangedFilesFromApi(token: string, pullRequest: PullRequestEve if (row.status === ChangeStatus.Renamed) { files.push({ filename: row.filename, - status: ChangeStatus.Added + status: ChangeStatus.Added, + additions: row.additions, + deletions: row.deletions }) files.push({ // 'previous_filename' for some unknown reason isn't in the type definition or documentation filename: (row).previous_filename as string, - status: ChangeStatus.Deleted + status: ChangeStatus.Deleted, + additions: row.additions, + deletions: row.deletions }) } else { // Github status and git status variants are same except for deleted files const status = row.status === 'removed' ? ChangeStatus.Deleted : (row.status as ChangeStatus) files.push({ filename: row.filename, - status + status, + additions: row.additions, + deletions: row.deletions }) } } @@ -228,12 +241,20 @@ async function getChangedFilesFromApi(token: string, pullRequest: PullRequestEve } } -function exportResults(results: FilterResults, format: ExportFormat): void { +interface Stat { + additionCount: number + deletionCount: number + fileCount: number +} + +function exportResults(results: FilterResults, filesFormat: FilesExportFormat, statFormat: StatExportFormat): void { core.info('Results:') - const changes = [] + const changes: string[] = [] + const changeStats: {[key: string]: Stat} = {} + for (const [key, files] of Object.entries(results)) { - const value = files.length > 0 - core.startGroup(`Filter ${key} = ${value}`) + const hasMatchingFiles = files.length > 0 + core.startGroup(`Filter ${key} = ${hasMatchingFiles}`) if (files.length > 0) { changes.push(key) core.info('Matching files:') @@ -244,12 +265,24 @@ function exportResults(results: FilterResults, format: ExportFormat): void { core.info('Matching files: none') } - core.setOutput(key, value) + core.setOutput(key, hasMatchingFiles) core.setOutput(`${key}_count`, files.length) - if (format !== 'none') { - const filesValue = serializeExport(files, format) + if (filesFormat !== 'none') { + const filesValue = serializeExportChangedFiles(files, filesFormat) core.setOutput(`${key}_files`, filesValue) } + + const additionCount: number = files.reduce((sum, f) => sum + f.additions, 0) + const deletionCount: number = files.reduce((sum, f) => sum + f.deletions, 0) + core.setOutput(`${key}_addition_count`, additionCount) + core.setOutput(`${key}_deletion_count`, deletionCount) + core.setOutput(`${key}_change_count`, additionCount + deletionCount) + changeStats[key] = { + additionCount, + deletionCount, + fileCount: files.length + } + core.endGroup() } @@ -260,9 +293,14 @@ function exportResults(results: FilterResults, format: ExportFormat): void { } else { core.info('Cannot set changes output variable - name already used by filter output') } + + if (statFormat !== 'none') { + const statValue = serializeExportStat(changeStats, statFormat) + core.setOutput(`stat`, statValue) + } } -function serializeExport(files: File[], format: ExportFormat): string { +function serializeExportChangedFiles(files: File[], format: FilesExportFormat): string { const fileNames = files.map(file => file.filename) switch (format) { case 'csv': @@ -278,7 +316,21 @@ function serializeExport(files: File[], format: ExportFormat): string { } } -function isExportFormat(value: string): value is ExportFormat { +function serializeExportStat(stat: {[key: string]: Stat}, format: StatExportFormat): string { + switch (format) { + case 'csv': + return Object.keys(stat) + .sort() + .map(k => [csvEscape(k), stat[k].additionCount, stat[k].deletionCount, stat[k].fileCount].join(',')) + .join('\n') + case 'json': + return JSON.stringify(stat) + default: + return '' + } +} + +function isFilesExportFormat(value: string): value is FilesExportFormat { return ['none', 'csv', 'shell', 'json', 'escape'].includes(value) } @@ -287,4 +339,8 @@ function getErrorMessage(error: unknown): string { return String(error) } +function isStatExportFormat(value: string): value is StatExportFormat { + return ['none', 'csv', 'json'].includes(value) +} + run()