Merge pull request #53 from dorny/detect-local-changes

Support local changes

.github/workflows/pull-request-verification.yml

@@ -71,6 +71,23 @@ jobs:
       if: steps.filter.outputs.any != 'true' || steps.filter.outputs.error == 'true'
       run: exit 1
 
+  test-local-changes:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: echo "NEW FILE" > local
+    - run: git add local
+    - uses: ./
+      id: filter
+      with:
+        base: HEAD
+        filters: |
+          local:
+            - local
+    - name: filter-test
+      if: steps.filter.outputs.local != 'true'
+      run: exit 1
+
   test-change-type:
     runs-on: ubuntu-latest
     steps:

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## v2.6.0
+- [Support local changes](https://github.com/dorny/paths-filter/pull/53)
+
+## v2.5.3
+- [Fixed mapping of removed/deleted change status from github API](https://github.com/dorny/paths-filter/pull/51)
+- [Fixed retrieval of all changes via Github API when there are 100+ changes](https://github.com/dorny/paths-filter/pull/50)
+
+## v2.5.2
+- [Add support for multiple patterns when using file status](https://github.com/dorny/paths-filter/pull/48)
+- [Use picomatch directly instead of micromatch wrapper](https://github.com/dorny/paths-filter/pull/49)
+
 ## v2.5.1
 - [Improved path matching with micromatch](https://github.com/dorny/paths-filter/pull/46)
 

README.md

@@ -35,6 +35,10 @@ doesn't allow this because they doesn't work on a level of individual jobs or st
   when `base` input parameter is same as the branch that triggered the workflow:
     - Changes are detected from last commit
   - Uses git commands to detect changes - repository must be already [checked out](https://github.com/actions/checkout)
+- **Local changes**
+  - Workflow triggered by any event when `base` input parameter is set to `HEAD`
+  - Changes are detected against current HEAD
+  - Untracked files are ignored
 
 ## Example
 ```yaml
@@ -52,9 +56,9 @@ doesn't allow this because they doesn't work on a level of individual jobs or st
 For more scenarios see [examples](#examples) section.
 
 ## Notes:
-- Paths expressions are evaluated using [micromatch](https://github.com/micromatch/micromatch) library.
+- Paths expressions are evaluated using [picomatch](https://github.com/micromatch/picomatch) library.
   Documentation for path expression format can be found on project github page.
-- Micromatch [dot](https://github.com/micromatch/micromatch#options) option is set to true.
+- Micromatch [dot](https://github.com/micromatch/picomatch#options) option is set to true.
   Globbing will match also paths where file or folder name starts with a dot.
 - It's recommended to quote your path expressions with `'` or `"`. Otherwise you will get an error if it starts with `*`.
 - Local execution with [act](https://github.com/nektos/act) works only with alternative runner image. Default runner doesn't have `git` binary.
@@ -62,7 +66,9 @@ For more scenarios see [examples](#examples) section.
 
 
 # What's New
-- Paths expressions are now evaluated using [micromatch](https://github.com/micromatch/micromatch) library
+- Support local changes
+- Fixed retrieval of all changes via Github API when there are 100+ changes
+- Paths expressions are now evaluated using [picomatch](https://github.com/micromatch/picomatch) library
 - Support workflows triggered by any event
 - Fixed compatibility with older (<2.23) versions of git
 - Support for tag pushes and tags as a base reference
@@ -303,6 +309,35 @@ jobs:
 ```
 </details>
 
+<details>
+  <summary><b>Local changes:</b> Detect staged and unstaged local changes</summary>
+
+```yaml
+on:
+  push:
+    branches: # Push to following branches will trigger the workflow
+      - master
+      - develop
+      - release/**
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+      # Some action which modifies files tracked by git (e.g. code linter)
+    - uses: johndoe/some-action@v1
+
+      # Filter to detect which files were modified
+      # Changes could be for example automatically committed
+    - uses: dorny/paths-filter@v2
+      id: filter
+      with:
+        base: HEAD
+        filters: ... # Configure your filters
+```
+</details>
+
 ## Advanced options
 
 <details>
@@ -350,10 +385,15 @@ jobs:
         # dictionary, where type(s) of change composes the key.
         # Multiple change types can be specified using `|` as delimiter.
         filters: |
+          shared: &shared
+            - common/**
+            - config/**
           addedOrModified:
             - added|modified: '**'
           allChanges:
             - added|deleted|modified: '**'
+          addedOrModifiedAnchors:
+            - added|modified: *shared
 ```
 </details>
 
@@ -405,7 +445,7 @@ jobs:
 - name: Lint Markdown
   uses: johndoe/some-action@v1
   with:
-    files: ${{ steps.filter.changed_files }}
+    files: ${{ steps.filter.outputs.changed_files }}
 ```
 </details>
 

__tests__/filter.test.ts

@@ -165,6 +165,20 @@ describe('matching specific change status', () => {
     const match = filter.match(files)
     expect(match.addOrModify).toEqual(files)
   })
+
+  test('matches when using an anchor', () => {
+    const yaml = `
+    shared: &shared
+      - common/**/*
+      - config/**/*
+    src:
+      - modified: *shared
+    `
+    let filter = new Filter(yaml)
+    const files = modified(['config/file.js', 'common/anotherFile.js'])
+    const match = filter.match(files)
+    expect(match.src).toEqual(files)
+  })
 })
 
 function modified(paths: string[]): File[] {

package-lock.json

@@ -3723,12 +3723,6 @@
         "@babel/types": "^7.3.0"
       }
     },
-    "@types/braces": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/@types/braces/-/braces-3.0.0.tgz",
-      "integrity": "sha512-TbH79tcyi9FHwbyboOKeRachRq63mSuWYXOflsNO9ZyE5ClQ/JaozNKl+aWUq87qPNsXasXxi2AbgfwIJ+8GQw==",
-      "dev": true
-    },
     "@types/color-name": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
@@ -3797,15 +3791,6 @@
       "integrity": "sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA==",
       "dev": true
     },
-    "@types/micromatch": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/@types/micromatch/-/micromatch-4.0.1.tgz",
-      "integrity": "sha512-my6fLBvpY70KattTNzYOK6KU1oR1+UCz9ug/JbcF5UrEmeCt9P7DV2t7L8+t18mMPINqGQCE4O8PLOPbI84gxw==",
-      "dev": true,
-      "requires": {
-        "@types/braces": "*"
-      }
-    },
     "@types/minimatch": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
@@ -3823,6 +3808,12 @@
       "integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==",
       "dev": true
     },
+    "@types/picomatch": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/@types/picomatch/-/picomatch-2.2.1.tgz",
+      "integrity": "sha512-26/tQcDmJXYHiaWAAIjnTVL5nwrT+IVaqFZIbBImAuKk/r/j1r/1hmZ7uaOzG6IknqP3QHcNNQ6QO8Vp28lUoA==",
+      "dev": true
+    },
     "@types/prettier": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.0.1.tgz",
@@ -4980,14 +4971,6 @@
         "concat-map": "0.0.1"
       }
     },
-    "braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
-      "requires": {
-        "fill-range": "^7.0.1"
-      }
-    },
     "browser-process-hrtime": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz",
@@ -6586,14 +6569,6 @@
         "flat-cache": "^2.0.1"
       }
     },
-    "fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
-      "requires": {
-        "to-regex-range": "^5.0.1"
-      }
-    },
     "find-up": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
@@ -7228,11 +7203,6 @@
         "is-extglob": "^2.1.1"
       }
     },
-    "is-number": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
-      "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="
-    },
     "is-plain-object": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
@@ -12570,15 +12540,6 @@
       "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
       "dev": true
     },
-    "micromatch": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz",
-      "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==",
-      "requires": {
-        "braces": "^3.0.1",
-        "picomatch": "^2.0.5"
-      }
-    },
     "mime-db": {
       "version": "1.44.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz",
@@ -14397,14 +14358,6 @@
         "safe-regex": "^1.1.0"
       }
     },
-    "to-regex-range": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
-      "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
-      "requires": {
-        "is-number": "^7.0.0"
-      }
-    },
     "tough-cookie": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz",

package.json

@@ -29,14 +29,14 @@
     "@actions/exec": "^1.0.4",
     "@actions/github": "^2.2.0",
     "@octokit/webhooks": "^7.6.2",
-    "micromatch": "^4.0.2"
+    "picomatch": "^2.2.2"
   },
   "devDependencies": {
     "@types/jest": "^25.2.3",
     "@types/js-yaml": "^3.12.4",
-    "@types/micromatch": "^4.0.1",
     "@types/minimatch": "^3.0.3",
     "@types/node": "^14.0.5",
+    "@types/picomatch": "^2.2.1",
     "@typescript-eslint/parser": "^3.3.0",
     "@zeit/ncc": "^0.22.3",
     "eslint": "^7.3.0",

src/filter.ts

@@ -1,5 +1,5 @@
 import * as jsyaml from 'js-yaml'
-import * as micromatch from 'micromatch'
+import picomatch from 'picomatch'
 import {File, ChangeStatus} from './file'
 
 // Type definition of object we expect to load from YAML
@@ -8,11 +8,11 @@ interface FilterYaml {
 }
 type FilterItemYaml =
   | string // Filename pattern, e.g. "path/to/*.js"
-  | {[changeTypes: string]: string} // Change status and filename, e.g. added|modified: "path/to/*.js"
+  | {[changeTypes: string]: string | string[]} // Change status and filename, e.g. added|modified: "path/to/*.js"
   | FilterItemYaml[] // Supports referencing another rule via YAML anchor
 
 // Minimatch options used in all matchers
-const MatchOptions: micromatch.Options = {
+const MatchOptions = {
   dot: true
 }
 
@@ -73,14 +73,14 @@ export class Filter {
     }
 
     if (typeof item === 'string') {
-      return [{status: undefined, isMatch: micromatch.matcher(item, MatchOptions)}]
+      return [{status: undefined, isMatch: picomatch(item, MatchOptions)}]
     }
 
     if (typeof item === 'object') {
       return Object.entries(item).map(([key, pattern]) => {
-        if (typeof key !== 'string' || typeof pattern !== 'string') {
+        if (typeof key !== 'string' || (typeof pattern !== 'string' && !Array.isArray(pattern))) {
           this.throwInvalidFormatError(
-            `Expected [key:string]= pattern:string, but [${key}:${typeof key}]= ${pattern}:${typeof pattern} found`
+            `Expected [key:string]= pattern:string | string[], but [${key}:${typeof key}]= ${pattern}:${typeof pattern} found`
           )
         }
         return {
@@ -89,7 +89,7 @@ export class Filter {
             .map(x => x.trim())
             .filter(x => x.length > 0)
             .map(x => x.toLowerCase()) as ChangeStatus[],
-          isMatch: micromatch.matcher(pattern, MatchOptions)
+          isMatch: picomatch(pattern, MatchOptions)
         }
       })
     }

src/git.ts

@@ -3,6 +3,7 @@ import * as core from '@actions/core'
 import {File, ChangeStatus} from './file'
 
 export const NULL_SHA = '0000000000000000000000000000000000000000'
+export const HEAD = 'HEAD'
 
 export async function getChangesInLastCommit(): Promise<File[]> {
   core.startGroup(`Change detection in last commit`)
@@ -39,6 +40,20 @@ export async function getChanges(ref: string): Promise<File[]> {
   return parseGitDiffOutput(output)
 }
 
+export async function getChangesOnHead(): Promise<File[]> {
+  // Get current changes - both staged and unstaged
+  core.startGroup(`Change detection on HEAD`)
+  let output = ''
+  try {
+    output = (await exec('git', ['diff', '--no-renames', '--name-status', '-z', 'HEAD'])).stdout
+  } finally {
+    fixStdOutNullTermination()
+    core.endGroup()
+  }
+
+  return parseGitDiffOutput(output)
+}
+
 export async function getChangesSinceMergeBase(ref: string, initialFetchDepth: number): Promise<File[]> {
   if (!(await hasCommit(ref))) {
     // Fetch and add base branch

src/main.ts

@@ -55,6 +55,12 @@ function getConfigFileContent(configPath: string): string {
 }
 
 async function getChangedFiles(token: string, base: string, initialFetchDepth: number): Promise<File[]> {
+  // if base is 'HEAD' only local uncommitted changes will be detected
+  // This is the simplest case as we don't need to fetch more commits or evaluate current/before refs
+  if (base === git.HEAD) {
+    return await git.getChangesOnHead()
+  }
+
   if (github.context.eventName === 'pull_request' || github.context.eventName === 'pull_request_target') {
     const pr = github.context.payload.pull_request as Webhooks.WebhookPayloadPullRequestPullRequest
     if (token) {
@@ -75,7 +81,7 @@ async function getChangedFilesFromGit(base: string, initialFetchDepth: number):
 
   const pushRef =
     git.getShortName(github.context.ref) ||
-    (core.warning(`'ref' field is missing in PUSH event payload - using current branch, tag or commit SHA`),
+    (core.warning(`'ref' field is missing in event payload - using current branch, tag or commit SHA`),
     await git.getCurrentRef())
 
   const baseRef = git.getShortName(base) || defaultRef
@@ -88,11 +94,11 @@ async function getChangedFilesFromGit(base: string, initialFetchDepth: number):
   const isBaseRefSha = git.isGitSha(baseRef)
   const isBaseSameAsPush = baseRef === pushRef
 
-  // If base is commit SHA will do comparison against the referenced commit
-  // Or If base references same branch it was pushed to, we will do comparison against the previously pushed commit
+  // If base is commit SHA we will do comparison against the referenced commit
+  // Or if base references same branch it was pushed to, we will do comparison against the previously pushed commit
   if (isBaseRefSha || isBaseSameAsPush) {
     if (!isBaseRefSha && !beforeSha) {
-      core.warning(`'before' field is missing in PUSH event payload - changes will be detected from last commit`)
+      core.warning(`'before' field is missing in event payload - changes will be detected from last commit`)
       return await git.getChangesInLastCommit()
     }
 
@@ -123,11 +129,13 @@ async function getChangedFilesFromApi(
   token: string,
   pullRequest: Webhooks.WebhookPayloadPullRequestPullRequest
 ): Promise<File[]> {
-  core.info(`Fetching list of changed files for PR#${pullRequest.number} from Github API`)
+  core.startGroup(`Fetching list of changed files for PR#${pullRequest.number} from Github API`)
+  core.info(`Number of changed_files is ${pullRequest.changed_files}`)
   const client = new github.GitHub(token)
   const pageSize = 100
   const files: File[] = []
-  for (let page = 0; page * pageSize < pullRequest.changed_files; page++) {
+  for (let page = 1; (page - 1) * pageSize < pullRequest.changed_files; page++) {
+    core.info(`Invoking listFiles(pull_number: ${pullRequest.number}, page: ${page}, per_page: ${pageSize})`)
     const response = await client.pulls.listFiles({
       owner: github.context.repo.owner,
       repo: github.context.repo.repo,
@@ -136,6 +144,7 @@ async function getChangedFilesFromApi(
       per_page: pageSize
     })
     for (const row of response.data) {
+      core.info(`[${row.status}] ${row.filename}`)
       // There's no obvious use-case for detection of renames
       // Therefore we treat it as if rename detection in git diff was turned off.
       // Rename is replaced by delete of original filename and add of new filename
@@ -150,14 +159,17 @@ async function getChangedFilesFromApi(
           status: ChangeStatus.Deleted
         })
       } else {
+        // Github status and git status variants are same except for deleted files
+        const status = row.status === 'removed' ? ChangeStatus.Deleted : (row.status as ChangeStatus)
         files.push({
           filename: row.filename,
-          status: row.status as ChangeStatus
+          status
         })
       }
     }
   }
 
+  core.endGroup()
   return files
 }