Fix change detection using Github API (#51)

fix: retrieve all changes via api

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## v2.5.3
+- [Fixed mapping of removed/deleted change status from github API](https://github.com/dorny/paths-filter/pull/51)
+- [Fixed retrieval of all changes via Github API when there are 100+ changes](https://github.com/dorny/paths-filter/pull/50)
+
+## v2.5.2
+- [Add support for multiple patterns when using file status](https://github.com/dorny/paths-filter/pull/48)
+- [Use picomatch directly instead of micromatch wrapper](https://github.com/dorny/paths-filter/pull/49)
+
 ## v2.5.1
 - [Improved path matching with micromatch](https://github.com/dorny/paths-filter/pull/46)
 

README.md

@@ -52,9 +52,9 @@ doesn't allow this because they doesn't work on a level of individual jobs or st
 For more scenarios see [examples](#examples) section.
 
 ## Notes:
-- Paths expressions are evaluated using [micromatch](https://github.com/micromatch/micromatch) library.
+- Paths expressions are evaluated using [picomatch](https://github.com/micromatch/picomatch) library.
   Documentation for path expression format can be found on project github page.
-- Micromatch [dot](https://github.com/micromatch/micromatch#options) option is set to true.
+- Micromatch [dot](https://github.com/micromatch/picomatch#options) option is set to true.
   Globbing will match also paths where file or folder name starts with a dot.
 - It's recommended to quote your path expressions with `'` or `"`. Otherwise you will get an error if it starts with `*`.
 - Local execution with [act](https://github.com/nektos/act) works only with alternative runner image. Default runner doesn't have `git` binary.
@@ -62,7 +62,8 @@ For more scenarios see [examples](#examples) section.
 
 
 # What's New
-- Paths expressions are now evaluated using [micromatch](https://github.com/micromatch/micromatch) library
+- Fixed retrieval of all changes via Github API when there are 100+ changes
+- Paths expressions are now evaluated using [picomatch](https://github.com/micromatch/picomatch) library
 - Support workflows triggered by any event
 - Fixed compatibility with older (<2.23) versions of git
 - Support for tag pushes and tags as a base reference
@@ -350,10 +351,15 @@ jobs:
         # dictionary, where type(s) of change composes the key.
         # Multiple change types can be specified using `|` as delimiter.
         filters: |
+          shared: &shared
+            - common/**
+            - config/**
           addedOrModified:
             - added|modified: '**'
           allChanges:
             - added|deleted|modified: '**'
+          addedOrModifiedAnchors:
+            - added|modified: *shared
 ```
 </details>
 
@@ -405,7 +411,7 @@ jobs:
 - name: Lint Markdown
   uses: johndoe/some-action@v1
   with:
-    files: ${{ steps.filter.changed_files }}
+    files: ${{ steps.filter.outputs.changed_files }}
 ```
 </details>
 

__tests__/filter.test.ts

@@ -165,6 +165,20 @@ describe('matching specific change status', () => {
     const match = filter.match(files)
     expect(match.addOrModify).toEqual(files)
   })
+
+  test('matches when using an anchor', () => {
+    const yaml = `
+    shared: &shared
+      - common/**/*
+      - config/**/*
+    src:
+      - modified: *shared
+    `
+    let filter = new Filter(yaml)
+    const files = modified(['config/file.js', 'common/anotherFile.js'])
+    const match = filter.match(files)
+    expect(match.src).toEqual(files)
+  })
 })
 
 function modified(paths: string[]): File[] {

package-lock.json

@@ -3723,12 +3723,6 @@
         "@babel/types": "^7.3.0"
       }
     },
-    "@types/braces": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/@types/braces/-/braces-3.0.0.tgz",
-      "integrity": "sha512-TbH79tcyi9FHwbyboOKeRachRq63mSuWYXOflsNO9ZyE5ClQ/JaozNKl+aWUq87qPNsXasXxi2AbgfwIJ+8GQw==",
-      "dev": true
-    },
     "@types/color-name": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
@@ -3797,15 +3791,6 @@
       "integrity": "sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA==",
       "dev": true
     },
-    "@types/micromatch": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/@types/micromatch/-/micromatch-4.0.1.tgz",
-      "integrity": "sha512-my6fLBvpY70KattTNzYOK6KU1oR1+UCz9ug/JbcF5UrEmeCt9P7DV2t7L8+t18mMPINqGQCE4O8PLOPbI84gxw==",
-      "dev": true,
-      "requires": {
-        "@types/braces": "*"
-      }
-    },
     "@types/minimatch": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
@@ -3823,6 +3808,12 @@
       "integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==",
       "dev": true
     },
+    "@types/picomatch": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/@types/picomatch/-/picomatch-2.2.1.tgz",
+      "integrity": "sha512-26/tQcDmJXYHiaWAAIjnTVL5nwrT+IVaqFZIbBImAuKk/r/j1r/1hmZ7uaOzG6IknqP3QHcNNQ6QO8Vp28lUoA==",
+      "dev": true
+    },
     "@types/prettier": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.0.1.tgz",
@@ -4980,14 +4971,6 @@
         "concat-map": "0.0.1"
       }
     },
-    "braces": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
-      "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
-      "requires": {
-        "fill-range": "^7.0.1"
-      }
-    },
     "browser-process-hrtime": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz",
@@ -6586,14 +6569,6 @@
         "flat-cache": "^2.0.1"
       }
     },
-    "fill-range": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
-      "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
-      "requires": {
-        "to-regex-range": "^5.0.1"
-      }
-    },
     "find-up": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
@@ -7228,11 +7203,6 @@
         "is-extglob": "^2.1.1"
       }
     },
-    "is-number": {
-      "version": "7.0.0",
-      "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
-      "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="
-    },
     "is-plain-object": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
@@ -12570,15 +12540,6 @@
       "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
       "dev": true
     },
-    "micromatch": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz",
-      "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==",
-      "requires": {
-        "braces": "^3.0.1",
-        "picomatch": "^2.0.5"
-      }
-    },
     "mime-db": {
       "version": "1.44.0",
       "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz",
@@ -14397,14 +14358,6 @@
         "safe-regex": "^1.1.0"
       }
     },
-    "to-regex-range": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
-      "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
-      "requires": {
-        "is-number": "^7.0.0"
-      }
-    },
     "tough-cookie": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz",

package.json

@@ -29,14 +29,14 @@
     "@actions/exec": "^1.0.4",
     "@actions/github": "^2.2.0",
     "@octokit/webhooks": "^7.6.2",
-    "micromatch": "^4.0.2"
+    "picomatch": "^2.2.2"
   },
   "devDependencies": {
     "@types/jest": "^25.2.3",
     "@types/js-yaml": "^3.12.4",
-    "@types/micromatch": "^4.0.1",
     "@types/minimatch": "^3.0.3",
     "@types/node": "^14.0.5",
+    "@types/picomatch": "^2.2.1",
     "@typescript-eslint/parser": "^3.3.0",
     "@zeit/ncc": "^0.22.3",
     "eslint": "^7.3.0",

src/filter.ts

@@ -1,5 +1,5 @@
 import * as jsyaml from 'js-yaml'
-import * as micromatch from 'micromatch'
+import picomatch from 'picomatch'
 import {File, ChangeStatus} from './file'
 
 // Type definition of object we expect to load from YAML
@@ -8,11 +8,11 @@ interface FilterYaml {
 }
 type FilterItemYaml =
   | string // Filename pattern, e.g. "path/to/*.js"
-  | {[changeTypes: string]: string} // Change status and filename, e.g. added|modified: "path/to/*.js"
+  | {[changeTypes: string]: string | string[]} // Change status and filename, e.g. added|modified: "path/to/*.js"
   | FilterItemYaml[] // Supports referencing another rule via YAML anchor
 
 // Minimatch options used in all matchers
-const MatchOptions: micromatch.Options = {
+const MatchOptions = {
   dot: true
 }
 
@@ -73,14 +73,14 @@ export class Filter {
     }
 
     if (typeof item === 'string') {
-      return [{status: undefined, isMatch: micromatch.matcher(item, MatchOptions)}]
+      return [{status: undefined, isMatch: picomatch(item, MatchOptions)}]
     }
 
     if (typeof item === 'object') {
       return Object.entries(item).map(([key, pattern]) => {
-        if (typeof key !== 'string' || typeof pattern !== 'string') {
+        if (typeof key !== 'string' || (typeof pattern !== 'string' && !Array.isArray(pattern))) {
           this.throwInvalidFormatError(
-            `Expected [key:string]= pattern:string, but [${key}:${typeof key}]= ${pattern}:${typeof pattern} found`
+            `Expected [key:string]= pattern:string | string[], but [${key}:${typeof key}]= ${pattern}:${typeof pattern} found`
           )
         }
         return {
@@ -89,7 +89,7 @@ export class Filter {
             .map(x => x.trim())
             .filter(x => x.length > 0)
             .map(x => x.toLowerCase()) as ChangeStatus[],
-          isMatch: micromatch.matcher(pattern, MatchOptions)
+          isMatch: picomatch(pattern, MatchOptions)
         }
       })
     }

src/main.ts

@@ -123,11 +123,13 @@ async function getChangedFilesFromApi(
   token: string,
   pullRequest: Webhooks.WebhookPayloadPullRequestPullRequest
 ): Promise<File[]> {
-  core.info(`Fetching list of changed files for PR#${pullRequest.number} from Github API`)
+  core.startGroup(`Fetching list of changed files for PR#${pullRequest.number} from Github API`)
+  core.info(`Number of changed_files is ${pullRequest.changed_files}`)
   const client = new github.GitHub(token)
   const pageSize = 100
   const files: File[] = []
-  for (let page = 0; page * pageSize < pullRequest.changed_files; page++) {
+  for (let page = 1; (page - 1) * pageSize < pullRequest.changed_files; page++) {
+    core.info(`Invoking listFiles(pull_number: ${pullRequest.number}, page: ${page}, per_page: ${pageSize})`)
     const response = await client.pulls.listFiles({
       owner: github.context.repo.owner,
       repo: github.context.repo.repo,
@@ -136,6 +138,7 @@ async function getChangedFilesFromApi(
       per_page: pageSize
     })
     for (const row of response.data) {
+      core.info(`[${row.status}] ${row.filename}`)
       // There's no obvious use-case for detection of renames
       // Therefore we treat it as if rename detection in git diff was turned off.
       // Rename is replaced by delete of original filename and add of new filename
@@ -150,14 +153,17 @@ async function getChangedFilesFromApi(
           status: ChangeStatus.Deleted
         })
       } else {
+        // Github status and git status variants are same except for deleted files
+        const status = row.status === 'removed' ? ChangeStatus.Deleted : (row.status as ChangeStatus)
         files.push({
           filename: row.filename,
-          status: row.status as ChangeStatus
+          status
         })
       }
     }
   }
 
+  core.endGroup()
   return files
 }