Merge pull request #68 from dorny/list-files-csv

Add list-files: csv format

.github/workflows/build.yml

@@ -1,6 +1,7 @@
 name: "Build"
 on:
   push:
+    paths-ignore: [ '*.md' ]
     branches:
       - master
 

.github/workflows/pull-request-verification.yml

@@ -1,6 +1,7 @@
 name: "Pull Request Verification"
 on:
   pull_request:
+    paths-ignore: [ '*.md' ]
     branches:
       - master
       - develop
@@ -29,6 +30,9 @@ jobs:
     - name: filter-test
       if: steps.filter.outputs.any != 'true' || steps.filter.outputs.error == 'true'
       run: exit 1
+    - name: changes-test
+      if: contains(fromJSON(steps.filter.outputs.changes), 'error') || !contains(fromJSON(steps.filter.outputs.changes), 'any')
+      run: exit 1
 
   test-external:
     runs-on: ubuntu-latest
@@ -87,6 +91,9 @@ jobs:
     - name: filter-test
       if: steps.filter.outputs.local != 'true'
       run: exit 1
+    - name: count-test
+      if: steps.filter.outputs.local_count != 1
+      run: exit 1
 
   test-change-type:
     runs-on: ubuntu-latest
@@ -119,15 +126,12 @@ jobs:
     - name: Print 'deleted_files'
       run: echo ${{steps.filter.outputs.deleted_files}}
     - name: filter-test
-      # only single quotes are supported in GH action literal
-      # single quote needs to be escaped with single quote
-      # '''add.txt''' resolves to string 'add.txt'
       if: |
         steps.filter.outputs.added != 'true'
         || steps.filter.outputs.deleted != 'true'
         || steps.filter.outputs.modified != 'true'
         || steps.filter.outputs.any != 'true'
-        || steps.filter.outputs.added_files != '''add.txt'''
-        || steps.filter.outputs.modified_files != '''LICENSE'''
-        || steps.filter.outputs.deleted_files != '''README.md'''
+        || steps.filter.outputs.added_files != 'add.txt'
+        || steps.filter.outputs.modified_files != 'LICENSE'
+        || steps.filter.outputs.deleted_files != 'README.md'
       run: exit 1

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## v2.9.0
+- [Add list-files: csv format](https://github.com/dorny/paths-filter/pull/68)
+
+## v2.8.0
+- [Add count output variable](https://github.com/dorny/paths-filter/pull/65)
+- [Fix log grouping of changes](https://github.com/dorny/paths-filter/pull/61)
+
+## v2.7.0
+- [Add "changes" output variable to support matrix job configuration](https://github.com/dorny/paths-filter/pull/59)
+- [Improved listing of matching files with `list-files: shell` and `list-files: escape` options](https://github.com/dorny/paths-filter/pull/58)
+
 ## v2.6.0
 - [Support local changes](https://github.com/dorny/paths-filter/pull/53)
 

README.md

@@ -5,7 +5,7 @@ based on the files modified by pull request, feature branch or in pushed commits
 
 It saves time and resources especially in monorepo setups, where you can run slow tasks (e.g. integration tests or deployments) only for changed components.
 Github workflows built-in [path filters](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onpushpull_requestpaths)
-doesn't allow this because they doesn't work on a level of individual jobs or steps.
+don't allow this because they don't work on a level of individual jobs or steps.
 
 **Real world usage examples:**
 - [sentry.io](https://sentry.io/) - [backend-test-py3.6.yml](https://github.com/getsentry/sentry/blob/ca0e43dc5602a9ab2e06d3f6397cc48fb5a78541/.github/workflows/backend-test-py3.6.yml#L32)
@@ -58,7 +58,7 @@ For more scenarios see [examples](#examples) section.
 ## Notes:
 - Paths expressions are evaluated using [picomatch](https://github.com/micromatch/picomatch) library.
   Documentation for path expression format can be found on project github page.
-- Micromatch [dot](https://github.com/micromatch/picomatch#options) option is set to true.
+- Picomatch [dot](https://github.com/micromatch/picomatch#options) option is set to true.
   Globbing will match also paths where file or folder name starts with a dot.
 - It's recommended to quote your path expressions with `'` or `"`. Otherwise you will get an error if it starts with `*`.
 - Local execution with [act](https://github.com/nektos/act) works only with alternative runner image. Default runner doesn't have `git` binary.
@@ -66,20 +66,15 @@ For more scenarios see [examples](#examples) section.
 
 
 # What's New
+- Add `list-files: csv` format
+- Configure matrix job to run for each folder with changes using `changes` output
+- Improved listing of matching files with `list-files: shell` and `list-files: escape` options
 - Support local changes
 - Fixed retrieval of all changes via Github API when there are 100+ changes
 - Paths expressions are now evaluated using [picomatch](https://github.com/micromatch/picomatch) library
 - Support workflows triggered by any event
-- Fixed compatibility with older (<2.23) versions of git
-- Support for tag pushes and tags as a base reference
-- Fixes for various edge cases when event payload is incomplete
-  - Supports local execution with [act](https://github.com/nektos/act)
-- Fixed behavior of feature branch workflow:
-  - Detects only changes introduced by feature branch. Later modifications on base branch are ignored
-- Filter by type of file change:
-  - Optionally consider if file was added, modified or deleted
 
-For more information see [CHANGELOG](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
+For more information see [CHANGELOG](https://github.com/dorny/paths-filter/blob/master/CHANGELOG.md)
 
 # Usage
 
@@ -127,9 +122,13 @@ For more information see [CHANGELOG](https://github.com/actions/checkout/blob/ma
 
     # Enables listing of files matching the filter:
     #   'none'  - Disables listing of matching files (default).
+    #   'csv'   - Coma separated list of filenames.
+    #             If needed it uses double quotes to wrap filename with unsafe characters.
     #   'json'  - Matching files paths are formatted as JSON array.
-    #   'shell' - Matching files paths are escaped and space-delimited.
-    #             Output is usable as command line argument list in linux shell.
+    #   'shell' - Space delimited list usable as command line argument list in Linux shell.
+    #             If needed it uses single or double quotes to wrap filename with unsafe characters.
+    #   'escape'- Space delimited list usable as command line argument list in Linux shell.
+    #             Backslash escapes every potentially unsafe character.
     # Default: none
     list-files: ''
 
@@ -150,7 +149,9 @@ For more information see [CHANGELOG](https://github.com/actions/checkout/blob/ma
 - For each filter it sets output variable named by the filter to the text:
    - `'true'` - if **any** of changed files matches any of filter rules
    - `'false'` - if **none** of changed files matches any of filter rules
+- For each filter it sets output variable with name `${FILTER_NAME}_count` to the count of matching files.
 - If enabled, for each filter it sets output variable with name `${FILTER_NAME}_files`. It will contain list of all files matching the filter.
+- `changes` - JSON array with names of all filters matching any of changed files.
 
 # Examples
 
@@ -234,6 +235,41 @@ jobs:
 ```
 </details>
 
+<details>
+<summary>Use change detection to configure matrix job</summary>
+
+```yaml
+jobs:
+  # JOB to run change detection
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      # Expose matched filters as job 'packages' output variable
+      packages: ${{ steps.filter.outputs.changes }}
+    steps:
+    # For pull requests it's not necessary to checkout the code
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        filters: |
+          package1: src/package1
+          package2: src/package2
+
+  # JOB to build and test each of modified packages
+  build:
+    needs: changes
+    strategy:
+      matrix:
+        # Parse JSON array containing names of all filters matching any of changed files
+        # e.g. ['package1', 'package2'] if both package folders contains changes
+        package: ${{ fromJson(needs.changes.outputs.packages) }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - ...
+```
+</details>
+
 ## Change detection workflows
 
 <details>
@@ -410,7 +446,7 @@ jobs:
     # Enable listing of files matching each filter.
     # Paths to files will be available in `${FILTER_NAME}_files` output variable.
     # Paths will be escaped and space-delimited.
-    # Output is usable as command line argument list in linux shell
+    # Output is usable as command line argument list in Linux shell
     list-files: shell
 
     # In this example changed files will be checked by linter.
@@ -449,6 +485,8 @@ jobs:
 ```
 </details>
 
+# See also
+- [test-reporter](https://github.com/dorny/test-reporter) - Displays test results from popular testing frameworks directly in GitHub
 
 # License
 

__tests__/csv-escape.test.ts

@@ -0,0 +1,23 @@
+import {csvEscape} from '../src/list-format/csv-escape'
+
+describe('csvEscape() backslash escapes every character except subset of definitely safe characters', () => {
+  test('simple filename should not be modified', () => {
+    expect(csvEscape('file.txt')).toBe('file.txt')
+  })
+
+  test('directory separator should be preserved and not escaped', () => {
+    expect(csvEscape('path/to/file.txt')).toBe('path/to/file.txt')
+  })
+
+  test('filename with spaces should be quoted', () => {
+    expect(csvEscape('file with space')).toBe('"file with space"')
+  })
+
+  test('filename with "," should be quoted', () => {
+    expect(csvEscape('file, with coma')).toBe('"file, with coma"')
+  })
+
+  test('Double quote should be escaped by another double quote', () => {
+    expect(csvEscape('file " with double quote')).toBe('"file "" with double quote"')
+  })
+})

__tests__/shell-escape.test.ts

@@ -1,16 +1,57 @@
-import shellEscape from '../src/shell-escape'
+import {backslashEscape, shellEscape} from '../src/list-format/shell-escape'
 
-test('simple path escaped', () => {
-  expect(shellEscape('file')).toBe("'file'")
+describe('escape() backslash escapes every character except subset of definitely safe characters', () => {
+  test('simple filename should not be modified', () => {
+    expect(backslashEscape('file.txt')).toBe('file.txt')
+  })
+
+  test('directory separator should be preserved and not escaped', () => {
+    expect(backslashEscape('path/to/file.txt')).toBe('path/to/file.txt')
+  })
+
+  test('spaces should be escaped with backslash', () => {
+    expect(backslashEscape('file with space')).toBe('file\\ with\\ space')
+  })
+
+  test('quotes should be escaped with backslash', () => {
+    expect(backslashEscape('file\'with quote"')).toBe('file\\\'with\\ quote\\"')
+  })
+
+  test('$variables should be escaped', () => {
+    expect(backslashEscape('$var')).toBe('\\$var')
+  })
 })
 
-test('path with space is wrapped with single quotes', () => {
-  expect(shellEscape('file with space')).toBe("'file with space'")
-})
+describe('shellEscape() returns human readable filenames with as few escaping applied as possible', () => {
+  test('simple filename should not be modified', () => {
+    expect(shellEscape('file.txt')).toBe('file.txt')
+  })
 
-test('path with quote is divided into quoted segments and escaped quote', () => {
-  expect(shellEscape("file'with quote")).toBe("'file'\\''with quote'")
-})
-test('path with leading quote does not have double quotes at beginning', () => {
-  expect(shellEscape("'file-leading-quote")).toBe("\\''file-leading-quote'")
+  test('directory separator should be preserved and not escaped', () => {
+    expect(shellEscape('path/to/file.txt')).toBe('path/to/file.txt')
+  })
+
+  test('filename with spaces should be quoted', () => {
+    expect(shellEscape('file with space')).toBe("'file with space'")
+  })
+
+  test('filename with spaces should be quoted', () => {
+    expect(shellEscape('file with space')).toBe("'file with space'")
+  })
+
+  test('filename with $ should be quoted', () => {
+    expect(shellEscape('$var')).toBe("'$var'")
+  })
+
+  test('filename with " should be quoted', () => {
+    expect(shellEscape('file"name')).toBe("'file\"name'")
+  })
+
+  test('filename with single quote should be wrapped in double quotes', () => {
+    expect(shellEscape("file'with quote")).toBe('"file\'with quote"')
+  })
+
+  test('filename with single quote and special characters is split and quoted/escaped as needed', () => {
+    expect(shellEscape("file'with $quote")).toBe("file\\''with $quote'")
+  })
 })

action.yml

@@ -22,8 +22,13 @@ inputs:
     description: |
       Enables listing of files matching the filter:
         'none'  - Disables listing of matching files (default).
-        'json'  - Matching files paths are serialized as JSON array.
-        'shell' - Matching files paths are escaped and space-delimited. Output is usable as command line argument list in linux shell.
+        'csv'   - Coma separated list of filenames.
+                  If needed it uses double quotes to wrap filename with unsafe characters.
+        'json'  - Serialized as JSON array.
+        'shell' - Space delimited list usable as command line argument list in linux shell.
+                  If needed it uses single or double quotes to wrap filename with unsafe characters.
+        'escape'- Space delimited list usable as command line argument list in linux shell.
+                  Backslash escapes every potentially unsafe character.
     required: true
     default: none
   initial-fetch-depth:
@@ -34,6 +39,9 @@ inputs:
       This option takes effect only when changes are detected using git against different base branch.
     required: false
     default: '10'
+outputs:
+  changes:
+    description: JSON array with names of all filters matching any of changed files
 runs:
   using: 'node12'
   main: 'dist/index.js'

dist/index.js

@@ -4641,9 +4641,6 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs = __importStar(__webpack_require__(747));
 const core = __importStar(__webpack_require__(470));
@@ -4651,7 +4648,8 @@ const github = __importStar(__webpack_require__(469));
 const filter_1 = __webpack_require__(235);
 const file_1 = __webpack_require__(258);
 const git = __importStar(__webpack_require__(136));
-const shell_escape_1 = __importDefault(__webpack_require__(751));
+const shell_escape_1 = __webpack_require__(206);
+const csv_escape_1 = __webpack_require__(410);
 async function run() {
     try {
         const workingDirectory = core.getInput('working-directory', { required: false });
@@ -4794,10 +4792,12 @@ async function getChangedFilesFromApi(token, pullRequest) {
 }
 function exportResults(results, format) {
     core.info('Results:');
+    const changes = [];
     for (const [key, files] of Object.entries(results)) {
         const value = files.length > 0;
         core.startGroup(`Filter ${key} = ${value}`);
         if (files.length > 0) {
+            changes.push(key);
             core.info('Matching files:');
             for (const file of files) {
                 core.info(`${file.filename} [${file.status}]`);
@@ -4807,26 +4807,39 @@ function exportResults(results, format) {
             core.info('Matching files: none');
         }
         core.setOutput(key, value);
+        core.setOutput(`${key}_count`, files.length);
         if (format !== 'none') {
             const filesValue = serializeExport(files, format);
             core.setOutput(`${key}_files`, filesValue);
         }
+        core.endGroup();
+    }
+    if (results['changes'] === undefined) {
+        const changesJson = JSON.stringify(changes);
+        core.info(`Changes output set to ${changesJson}`);
+        core.setOutput('changes', changesJson);
+    }
+    else {
+        core.info('Cannot set changes output variable - name already used by filter output');
     }
-    core.endGroup();
 }
 function serializeExport(files, format) {
     const fileNames = files.map(file => file.filename);
     switch (format) {
+        case 'csv':
+            return fileNames.map(csv_escape_1.csvEscape).join(',');
         case 'json':
             return JSON.stringify(fileNames);
+        case 'escape':
+            return fileNames.map(shell_escape_1.backslashEscape).join(' ');
         case 'shell':
-            return fileNames.map(shell_escape_1.default).join(' ');
+            return fileNames.map(shell_escape_1.shellEscape).join(' ');
         default:
             return '';
     }
 }
 function isExportFormat(value) {
-    return value === 'none' || value === 'shell' || value === 'json';
+    return ['none', 'csv', 'shell', 'json', 'escape'].includes(value);
 }
 run();
 
@@ -5018,6 +5031,43 @@ module.exports = {
 };
 
 
+/***/ }),
+
+/***/ 206:
+/***/ (function(__unusedmodule, exports) {
+
+"use strict";
+
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.shellEscape = exports.backslashEscape = void 0;
+// Backslash escape every character except small subset of definitely safe characters
+function backslashEscape(value) {
+    return value.replace(/([^a-zA-Z0-9,._+:@%/-])/gm, '\\$1');
+}
+exports.backslashEscape = backslashEscape;
+// Returns filename escaped for usage as shell argument.
+// Applies "human readable" approach with as few escaping applied as possible
+function shellEscape(value) {
+    if (value === '')
+        return value;
+    // Only safe characters
+    if (/^[a-zA-Z0-9,._+:@%/-]+$/m.test(value)) {
+        return value;
+    }
+    if (value.includes("'")) {
+        // Only safe characters, single quotes and white-spaces
+        if (/^[a-zA-Z0-9,._+:@%/'\s-]+$/m.test(value)) {
+            return `"${value}"`;
+        }
+        // Split by single quote and apply escaping recursively
+        return value.split("'").map(shellEscape).join("\\'");
+    }
+    // Contains some unsafe characters but no single quote
+    return `'${value}'`;
+}
+exports.shellEscape = shellEscape;
+
+
 /***/ }),
 
 /***/ 211:
@@ -8803,6 +8853,33 @@ function Octokit(plugins, options) {
 }
 
 
+/***/ }),
+
+/***/ 410:
+/***/ (function(__unusedmodule, exports) {
+
+"use strict";
+
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.csvEscape = void 0;
+// Returns filename escaped for CSV
+// Wraps file name into "..." only when it contains some potentially unsafe character
+function csvEscape(value) {
+    if (value === '')
+        return value;
+    // Only safe characters
+    if (/^[a-zA-Z0-9._+:@%/-]+$/m.test(value)) {
+        return value;
+    }
+    // https://tools.ietf.org/html/rfc4180
+    // If double-quotes are used to enclose fields, then a double-quote
+    // appearing inside a field must be escaped by preceding it with
+    // another double quote
+    return `"${value.replace(/"/g, '""')}"`;
+}
+exports.csvEscape = csvEscape;
+
+
 /***/ }),
 
 /***/ 413:
@@ -15215,23 +15292,6 @@ function sync (path, options) {
 
 module.exports = require("fs");
 
-/***/ }),
-
-/***/ 751:
-/***/ (function(__unusedmodule, exports) {
-
-"use strict";
-
-// Credits to https://github.com/xxorax/node-shell-escape
-Object.defineProperty(exports, "__esModule", { value: true });
-function shellEscape(value) {
-    return `'${value.replace(/'/g, "'\\''")}'`
-        .replace(/^(?:'')+/g, '') // unduplicate single-quote at the beginning
-        .replace(/\\'''/g, "\\'"); // remove non-escaped single-quote if there are enclosed between 2 escaped
-}
-exports.default = shellEscape;
-
-
 /***/ }),
 
 /***/ 753:

src/list-format/csv-escape.ts

@@ -0,0 +1,16 @@
+// Returns filename escaped for CSV
+// Wraps file name into "..." only when it contains some potentially unsafe character
+export function csvEscape(value: string): string {
+  if (value === '') return value
+
+  // Only safe characters
+  if (/^[a-zA-Z0-9._+:@%/-]+$/m.test(value)) {
+    return value
+  }
+
+  // https://tools.ietf.org/html/rfc4180
+  // If double-quotes are used to enclose fields, then a double-quote
+  // appearing inside a field must be escaped by preceding it with
+  // another double quote
+  return `"${value.replace(/"/g, '""')}"`
+}

src/list-format/shell-escape.ts

@@ -0,0 +1,28 @@
+// Backslash escape every character except small subset of definitely safe characters
+export function backslashEscape(value: string): string {
+  return value.replace(/([^a-zA-Z0-9,._+:@%/-])/gm, '\\$1')
+}
+
+// Returns filename escaped for usage as shell argument.
+// Applies "human readable" approach with as few escaping applied as possible
+export function shellEscape(value: string): string {
+  if (value === '') return value
+
+  // Only safe characters
+  if (/^[a-zA-Z0-9,._+:@%/-]+$/m.test(value)) {
+    return value
+  }
+
+  if (value.includes("'")) {
+    // Only safe characters, single quotes and white-spaces
+    if (/^[a-zA-Z0-9,._+:@%/'\s-]+$/m.test(value)) {
+      return `"${value}"`
+    }
+
+    // Split by single quote and apply escaping recursively
+    return value.split("'").map(shellEscape).join("\\'")
+  }
+
+  // Contains some unsafe characters but no single quote
+  return `'${value}'`
+}

src/main.ts

@@ -6,9 +6,10 @@ import {Webhooks} from '@octokit/webhooks'
 import {Filter, FilterResults} from './filter'
 import {File, ChangeStatus} from './file'
 import * as git from './git'
-import shellEscape from './shell-escape'
+import {backslashEscape, shellEscape} from './list-format/shell-escape'
+import {csvEscape} from './list-format/csv-escape'
 
-type ExportFormat = 'none' | 'json' | 'shell'
+type ExportFormat = 'none' | 'csv' | 'json' | 'shell' | 'escape'
 
 async function run(): Promise<void> {
   try {
@@ -175,10 +176,12 @@ async function getChangedFilesFromApi(
 
 function exportResults(results: FilterResults, format: ExportFormat): void {
   core.info('Results:')
+  const changes = []
   for (const [key, files] of Object.entries(results)) {
     const value = files.length > 0
     core.startGroup(`Filter ${key} = ${value}`)
     if (files.length > 0) {
+      changes.push(key)
       core.info('Matching files:')
       for (const file of files) {
         core.info(`${file.filename} [${file.status}]`)
@@ -188,19 +191,32 @@ function exportResults(results: FilterResults, format: ExportFormat): void {
     }
 
     core.setOutput(key, value)
+    core.setOutput(`${key}_count`, files.length)
     if (format !== 'none') {
       const filesValue = serializeExport(files, format)
       core.setOutput(`${key}_files`, filesValue)
     }
+    core.endGroup()
+  }
+
+  if (results['changes'] === undefined) {
+    const changesJson = JSON.stringify(changes)
+    core.info(`Changes output set to ${changesJson}`)
+    core.setOutput('changes', changesJson)
+  } else {
+    core.info('Cannot set changes output variable - name already used by filter output')
   }
-  core.endGroup()
 }
 
 function serializeExport(files: File[], format: ExportFormat): string {
   const fileNames = files.map(file => file.filename)
   switch (format) {
+    case 'csv':
+      return fileNames.map(csvEscape).join(',')
     case 'json':
       return JSON.stringify(fileNames)
+    case 'escape':
+      return fileNames.map(backslashEscape).join(' ')
     case 'shell':
       return fileNames.map(shellEscape).join(' ')
     default:
@@ -209,7 +225,7 @@ function serializeExport(files: File[], format: ExportFormat): string {
 }
 
 function isExportFormat(value: string): value is ExportFormat {
-  return value === 'none' || value === 'shell' || value === 'json'
+  return ['none', 'csv', 'shell', 'json', 'escape'].includes(value)
 }
 
 run()

src/shell-escape.ts

@@ -1,7 +0,0 @@
-// Credits to https://github.com/xxorax/node-shell-escape
-
-export default function shellEscape(value: string): string {
-  return `'${value.replace(/'/g, "'\\''")}'`
-    .replace(/^(?:'')+/g, '') // unduplicate single-quote at the beginning
-    .replace(/\\'''/g, "\\'") // remove non-escaped single-quote if there are enclosed between 2 escaped
-}