Improve change detection for feature branches (#16)

* Detect changes against configured base branch

* Update README and action.yml

* Add job.outputs example

* Update CHANGELOG

.eslintrc.json

@@ -25,7 +25,7 @@
       "@typescript-eslint/generic-type-naming": ["error", "^[A-Z][A-Za-z]*$"],
       "@typescript-eslint/no-array-constructor": "error",
       "@typescript-eslint/no-empty-interface": "error",
-      "@typescript-eslint/no-explicit-any": "error",
+      "@typescript-eslint/no-explicit-any": "off",
       "@typescript-eslint/no-extraneous-class": "error",
       "@typescript-eslint/no-for-in-array": "error",
       "@typescript-eslint/no-inferrable-types": "error",
@@ -42,7 +42,7 @@
       "@typescript-eslint/prefer-includes": "error",
       "@typescript-eslint/prefer-interface": "error",
       "@typescript-eslint/prefer-string-starts-ends-with": "error",
-      "@typescript-eslint/promise-function-async": "error",
+      "@typescript-eslint/promise-function-async": ["error", { "allowAny": true }],
       "@typescript-eslint/require-array-sort-compare": "error",
       "@typescript-eslint/restrict-plus-operands": "error",
       "semi": "off",

.github/filters.yml

@@ -0,0 +1,4 @@
+error:
+  - not_existing_path/**/*
+any:
+  - "**/*"

.github/workflows/build.yml

@@ -1,9 +1,6 @@
-name: "build-test"
+name: "Build"
 on:
-  pull_request:
-    types:
-      - opened
-      - synchronize
+  push:
     branches:
       - master
 
@@ -16,21 +13,14 @@ jobs:
         npm install
         npm run all
 
-  test:
+  self-test:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
     - uses: ./
       id: filter
       with:
-        githubToken: ${{ github.token }}
-        filters: |
-          src:
-            - src/**/*
-          tests:
-            - __tests__/**/*
-          any:
-            - "**/*"
+        filters: '.github/filters.yml'
     - name: filter-test
-      if: steps.filter.outputs.any != 'true'
+      if: steps.filter.outputs.any != 'true' || steps.filter.outputs.error == 'true'
       run: exit 1

.github/workflows/pull-request-verification.yml

@@ -0,0 +1,55 @@
+name: "Pull Request Verification"
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: |
+        npm install
+        npm run all
+
+  test-inline:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ./
+      id: filter
+      with:
+        filters: |
+          error:
+            - not_existing_path/**/*
+          any:
+            - "**/*"
+    - name: filter-test
+      if: steps.filter.outputs.any != 'true' || steps.filter.outputs.error == 'true'
+      run: exit 1
+
+  test-external:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ./
+      id: filter
+      with:
+        filters: '.github/filters.yml'
+    - name: filter-test
+      if: steps.filter.outputs.any != 'true' || steps.filter.outputs.error == 'true'
+      run: exit 1
+
+  test-without-token:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ./
+      id: filter
+      with:
+        token: ''
+        filters: '.github/filters.yml'
+    - name: filter-test
+      if: steps.filter.outputs.any != 'true' || steps.filter.outputs.error == 'true'
+      run: exit 1

CHANGELOG.md

@@ -0,0 +1,21 @@
+# Changelog
+
+## v2.2.0
+- [Improve change detection for feature branches](https://github.com/dorny/paths-filter/pull/16)
+
+## v2.1.0
+- [Support reusable paths blocks with yaml anchors](https://github.com/dorny/paths-filter/pull/13)
+
+## v2.0.0
+- [Added support for workflows triggered by push events](https://github.com/dorny/paths-filter/pull/10)
+- Action and repository renamed to paths-filter - original name doesn't make sense anymore
+
+## v1.1.0
+- [Allows filters to be specified in own .yml file](https://github.com/dorny/paths-filter/pull/8)
+- [Adds alternative change detection using git fetch and git diff-index](https://github.com/dorny/paths-filter/pull/9)
+
+## v1.0.1
+Updated dependencies - fixes github security alert
+
+## v1.0.0
+First official release uploaded to marketplace.

LICENSE

@@ -1,7 +1,7 @@
 
 The MIT License (MIT)
 
-Copyright (c) 2018 GitHub, Inc. and contributors
+Copyright (c) 2020 Michal Dorner and contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,50 +1,55 @@
 <p align="center">
-  <a href="https://github.com/dorny/pr-changed-files-filter/actions"><img alt="typescript-action status" src="https://github.com/dorny/pr-changed-files-filter/workflows/build-test/badge.svg"></a>
+  <a href="https://github.com/dorny/paths-filter/actions"><img alt="paths-filter status" src="https://github.com/dorny/paths-filter/workflows/Build/badge.svg"></a>
 </p>
 
-**CAUTION**: This action can be only used in a workflow triggered by `pull_request` event.
+# Paths filter
 
-# Pull request changed files filter
+With this [Github Action](https://github.com/features/actions) you can execute your workflow steps only if relevant files are modified.
 
-This [Github Action](https://github.com/features/actions) enables conditional execution of workflow job steps considering which files are modified by a pull request.
-
-It saves time and resources especially in monorepo setups, where you can run slow tasks (e.g. integration tests) only for changed components.
-Github workflows built-in
-[path filters](https://help.github.com/en/actions/referenceworkflow-syntax-for-github-actions#onpushpull_requestpaths)
+It saves time and resources especially in monorepo setups, where you can run slow tasks (e.g. integration tests or deployments) only for changed components.
+Github workflows built-in [path filters](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#onpushpull_requestpaths)
 doesn't allow this because they doesn't work on a level of individual jobs or steps.
 
+Supported workflows:
+- Action triggered by **[pull_request](https://help.github.com/en/actions/reference/events-that-trigger-workflows#pull-request-event-pull_request)** event:
+  - changes detected against the pull request base branch
+- Action triggered by **[push](https://help.github.com/en/actions/reference/events-that-trigger-workflows#push-event-push)** event:
+  - changes detected against the most recent commit on the same branch before the push
+  - changes detected against the top of the configured *base* branch (e.g. master)
+
 ## Usage
 
-The action accepts filter rules in the YAML format.
+Filter rules are defined using YAML format.
 Each filter rule is a list of [glob expressions](https://github.com/isaacs/minimatch).
 Corresponding output variable will be created to indicate if there's a changed file matching any of the rule glob expressions.
 Output variables can be later used in the `if` clause to conditionally run specific steps.
 
 ### Inputs
-- **`githubToken`**: GitHub Access Token - use `${{ github.token }}`
-- **`filters`**: YAML dictionary where keys specifies rule names and values are lists of file path patterns
+- **`token`**: GitHub Access Token - defaults to `${{ github.token }}` so you don't have to explicitly provide it.
+- **`base`**: Git reference (e.g. branch name) against which the changes will be detected. Defaults to repository default branch (e.g. master).
+              If it references same branch it was pushed to, changes are detected against the most recent commit before the push.
+              This option is ignored if action is triggered by *pull_request* event.
+- **`filters`**: Path to the configuration file or directly embedded string in YAML format. Filter configuration is a dictionary, where keys specifies rule names and values are lists of file path patterns.
 
 ### Outputs
 - For each rule it sets output variable named by the rule to text:
    - `'true'` - if **any** of changed files matches any of rule patterns
    - `'false'` - if **none** of changed files matches any of rule patterns
 
-
 ### Notes
 - minimatch [dot](https://www.npmjs.com/package/minimatch#dot) option is set to true - therefore
   globbing will match also paths where file or folder name starts with a dot.
+- You can use YAML anchors to reuse path expression(s) inside another rule. See example in the tests.
+- If changes are detected against the previous commit and there is none (i.e. first push of a new branch), all filter rules will report changed files.
+- You can use `base: ${{ github.ref }}` to configure change detection against previous commit for every branch you create.
 
-### Sample workflow
+### Example
 ```yaml
-...
-name: Build verification
-
 on:
+  push:
+    branches:
+      - master
   pull_request:
-    types:
-      - opened
-      - edited
-      - synchronize
     branches:
       - master
 jobs:
@@ -52,10 +57,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - uses: dorny/pr-changed-files-filter@v1
+    - uses: dorny/paths-filter@v2.2.0
       id: filter
       with:
-        githubToken: ${{ github.token }}
+        # inline YAML or path to separate file (e.g.: .github/filters.yaml)
         filters: |
           backend:
             - 'backend/**/*'
@@ -78,19 +83,55 @@ jobs:
       run: ...
 ```
 
+If your workflow uses multiple jobs, you can put *paths-filter* into own job and use
+[job outputs](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjobs_idoutputs)
+in other jobs [if](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idif) statements:
+```yml
+on:
+  pull_request:
+    branches:
+      - master
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    # Set job outputs to values from filter step
+    outputs:
+      backend: ${{ steps.filter.outputs.backend }}
+      frontend: ${{ steps.filter.outputs.frontend }}
+    steps:
+    # For pull requests it's not necessary to checkout the code
+    - uses: dorny/paths-filter@v2.2.0
+      id: filter
+      with:
+        # Filters stored in own yaml file
+        filters: '.github/filters.yml'
+  backend:
+    if: ${{ needs.changes.outputs.backend == 'true' }}
+    steps:
+      - ...
+  frontend:
+    if: ${{ needs.changes.outputs.frontend == 'true' }}
+    steps:
+      - ...
+```
+
 ## How it works
 
-1. Required inputs are checked (`githubToken` & `filters`)
-2. Provided access token is used to fetch list of changed files.
+1. If action was triggered by pull request:
+   - If access token was provided it's used to fetch list of changed files from Github API.
+   - If access token was not provided, top of the base branch is fetched and changed files are detected using `git diff-index <SHA>` command.
+2. If action was triggered by push event
+   - if *base* input parameter references same branch it was pushed to, most recent commit before the push is fetched
+   - If *base* input parameter references other branch, top of that branch is fetched
+   - changed files are detected using `git diff-index FETCH_HEAD` command.
 3. For each filter rule it checks if there is any matching file
 4. Output variables are set
 
-## Difference from related projects:
+## Difference from similar projects:
 
 - [Has Changed Path](https://github.com/MarceloPrado/has-changed-path)
   - detects changes from previous commit
   - you have to configure `checkout` action to fetch some number of previous commits
-  - `git diff` is used for change detection
   - outputs only single `true` / `false` value if any of provided paths contains changes
 - [Changed Files Exporter](https://github.com/futuratrepadeira/changed-files)
   - outputs lists with paths of created, updated and deleted files
@@ -98,4 +139,4 @@ jobs:
 - [Changed File Filter](https://github.com/tony84727/changed-file-filter)
   - allows change detection between any refs or commits
   - fetches whole history of your git repository
-  - might have negative performance impact on big repositories (github by default fetches only single commit)
+  - might have negative performance impact on big repositories (github by default fetches only single commit)

__tests__/filter.test.ts

@@ -90,4 +90,18 @@ describe('matching tests', () => {
     const match = filter.match(['.test/.test.js'])
     expect(match.dot).toBeTruthy()
   })
+
+  test('matches path based on rules included using YAML anchor', () => {
+    const yaml = `
+    shared: &shared
+      - common/**/*
+      - config/**/*
+    src:
+      - *shared
+      - src/**/*
+    `
+    let filter = new Filter(yaml)
+    const match = filter.match(['config/settings.yml'])
+    expect(match.src).toBeTruthy()
+  })
 })

__tests__/git.test.ts

@@ -0,0 +1,19 @@
+import * as git from '../src/git'
+
+describe('git utility function tests (those not invoking git)', () => {
+  test('Detects if ref references a tag', () => {
+    expect(git.isTagRef('refs/tags/v1.0')).toBeTruthy()
+    expect(git.isTagRef('refs/heads/master')).toBeFalsy()
+    expect(git.isTagRef('master')).toBeFalsy()
+  })
+  test('Trims "refs/" from ref', () => {
+    expect(git.trimRefs('refs/heads/master')).toBe('heads/master')
+    expect(git.trimRefs('heads/master')).toBe('heads/master')
+    expect(git.trimRefs('master')).toBe('master')
+  })
+  test('Trims "refs/" and "heads/" from ref', () => {
+    expect(git.trimRefsHeads('refs/heads/master')).toBe('master')
+    expect(git.trimRefsHeads('heads/master')).toBe('master')
+    expect(git.trimRefsHeads('master')).toBe('master')
+  })
+})

action.yml

@@ -1,12 +1,19 @@
-name: 'Pull request changed files filter'
-description: 'Enables conditional execution of workflow job steps considering which files are modified by a pull request.'
+name: 'Paths filter'
+description: 'Execute your workflow steps only if relevant files are modified.'
 author: 'Michal Dorner <dorner.michal@gmail.com>'
 inputs:
-  githubToken:
+  token:
     description: 'GitHub Access Token'
-    required: true
+    required: false
+    default: ${{ github.token }}
+  base:
+    description: |
+      Git reference (e.g. branch name) against which the changes will be detected. Defaults to repository default branch (e.g. master).
+      If it references same branch it was pushed to, changes are detected against the most recent commit before the push.
+      This option is ignored if action is triggered by pull_request event.
+    required: false
   filters:
-    description: 'YAML dictionary where keys specifies rule names and values are lists of (globbing) file path patterns'
+    description: 'Path to the configuration file or YAML string with filters definition'
     required: true
 runs:
   using: 'node12'

package.json

@@ -1,8 +1,8 @@
 {
-  "name": "pr-changed-files-filter",
+  "name": "paths-filter",
   "version": "1.0.0",
   "private": true,
-  "description": "Enables conditional execution of workflow job steps considering which files are modified by a pull request.",
+  "description": "Execute your workflow steps only if relevant files are modified.",
   "main": "lib/main.js",
   "scripts": {
     "build": "tsc",
@@ -25,27 +25,28 @@
   "author": "YourNameOrOrganization",
   "license": "MIT",
   "dependencies": {
-    "@actions/core": "^1.2.0",
+    "@actions/core": "^1.2.4",
+    "@actions/exec": "^1.0.4",
     "@actions/github": "^2.2.0",
-    "@octokit/webhooks": "^7.6.1",
+    "@octokit/webhooks": "^7.6.2",
     "minimatch": "^3.0.4"
   },
   "devDependencies": {
-    "@types/jest": "^24.0.23",
+    "@types/jest": "^25.2.3",
     "@types/js-yaml": "^3.12.4",
     "@types/minimatch": "^3.0.3",
-    "@types/node": "^12.7.12",
-    "@typescript-eslint/parser": "^2.8.0",
-    "@zeit/ncc": "^0.22.2",
-    "eslint": "^5.16.0",
+    "@types/node": "^14.0.5",
+    "@typescript-eslint/parser": "^3.3.0",
+    "@zeit/ncc": "^0.22.3",
+    "eslint": "^7.3.0",
     "eslint-plugin-github": "^2.0.0",
     "eslint-plugin-jest": "^22.21.0",
-    "jest": "^24.9.0",
-    "jest-circus": "^24.9.0",
-    "js-yaml": "^3.13.1",
-    "prettier": "^1.19.1",
-    "ts-jest": "^24.2.0",
-    "typescript": "^3.6.4"
+    "jest": "^26.0.1",
+    "jest-circus": "^26.0.1",
+    "js-yaml": "^3.14.0",
+    "prettier": "^2.0.5",
+    "ts-jest": "^26.0.0",
+    "typescript": "^3.9.3"
   },
   "jest": {
     "testEnvironment": "node"

src/filter.ts

@@ -15,10 +15,11 @@ export default class Filter {
     }
 
     for (const name of Object.keys(doc)) {
-      const patterns = doc[name] as string[]
-      if (!Array.isArray(patterns)) {
+      const patternsNode = doc[name]
+      if (!Array.isArray(patternsNode)) {
         this.throwInvalidFormatError()
       }
+      const patterns = flat(patternsNode) as string[]
       if (!patterns.every(x => typeof x === 'string')) {
         this.throwInvalidFormatError()
       }
@@ -40,3 +41,9 @@ export default class Filter {
     throw new Error('Invalid filter YAML format: Expected dictionary of string arrays')
   }
 }
+
+// Creates a new array with all sub-array elements recursively concatenated
+// In future could be replaced by Array.prototype.flat (supported on Node.js 11+)
+function flat(arr: any[]): any[] {
+  return arr.reduce((acc, val) => acc.concat(Array.isArray(val) ? flat(val) : val), [])
+}

src/git.ts

@@ -0,0 +1,46 @@
+import {exec} from '@actions/exec'
+
+export const NULL_SHA = '0000000000000000000000000000000000000000'
+export const FETCH_HEAD = 'FETCH_HEAD'
+
+export async function fetchCommit(ref: string): Promise<void> {
+  const exitCode = await exec('git', ['fetch', '--depth=1', '--no-tags', 'origin', ref])
+  if (exitCode !== 0) {
+    throw new Error(`Fetching ${ref} failed`)
+  }
+}
+
+export async function getChangedFiles(ref: string): Promise<string[]> {
+  let output = ''
+  const exitCode = await exec('git', ['diff-index', '--name-only', ref], {
+    listeners: {
+      stdout: (data: Buffer) => (output += data.toString())
+    }
+  })
+
+  if (exitCode !== 0) {
+    throw new Error(`Couldn't determine changed files`)
+  }
+
+  return output
+    .split('\n')
+    .map(s => s.trim())
+    .filter(s => s.length > 0)
+}
+
+export function isTagRef(ref: string): boolean {
+  return ref.startsWith('refs/tags/')
+}
+
+export function trimRefs(ref: string): string {
+  return trimStart(ref, 'refs/')
+}
+
+export function trimRefsHeads(ref: string): string {
+  const trimRef = trimStart(ref, 'refs/')
+  return trimStart(trimRef, 'heads/')
+}
+
+function trimStart(ref: string, start: string): string {
+  return ref.startsWith(start) ? ref.substr(start.length) : ref
+}

src/main.ts

@@ -1,38 +1,101 @@
+import * as fs from 'fs'
 import * as core from '@actions/core'
 import * as github from '@actions/github'
 import {Webhooks} from '@octokit/webhooks'
 
 import Filter from './filter'
+import * as git from './git'
 
 async function run(): Promise<void> {
   try {
-    const token = core.getInput('githubToken', {required: true})
-    const filterYaml = core.getInput('filters', {required: true})
-    const client = new github.GitHub(token)
+    const token = core.getInput('token', {required: false})
+    const filtersInput = core.getInput('filters', {required: true})
+    const filtersYaml = isPathInput(filtersInput) ? getConfigFileContent(filtersInput) : filtersInput
 
-    if (github.context.eventName !== 'pull_request') {
-      core.setFailed('This action can be triggered only by pull_request event')
-      return
-    }
+    const filter = new Filter(filtersYaml)
+    const files = await getChangedFiles(token)
 
-    const pr = github.context.payload.pull_request as Webhooks.WebhookPayloadPullRequestPullRequest
-    const filter = new Filter(filterYaml)
-    const files = await getChangedFiles(client, pr)
-
-    const result = filter.match(files)
-    for (const key in result) {
-      core.setOutput(key, String(result[key]))
+    if (files === null) {
+      // Change detection was not possible
+      // Set all filter keys to true (i.e. changed)
+      for (const key in filter.rules) {
+        core.setOutput(key, String(true))
+      }
+    } else {
+      const result = filter.match(files)
+      for (const key in result) {
+        core.setOutput(key, String(result[key]))
+      }
     }
   } catch (error) {
     core.setFailed(error.message)
   }
 }
 
+function isPathInput(text: string): boolean {
+  return !text.includes('\n')
+}
+
+function getConfigFileContent(configPath: string): string {
+  if (!fs.existsSync(configPath)) {
+    throw new Error(`Configuration file '${configPath}' not found`)
+  }
+
+  if (!fs.lstatSync(configPath).isFile()) {
+    throw new Error(`'${configPath}' is not a file.`)
+  }
+
+  return fs.readFileSync(configPath, {encoding: 'utf8'})
+}
+
+async function getChangedFiles(token: string): Promise<string[] | null> {
+  if (github.context.eventName === 'pull_request') {
+    const pr = github.context.payload.pull_request as Webhooks.WebhookPayloadPullRequestPullRequest
+    return token ? await getChangedFilesFromApi(token, pr) : await getChangedFilesFromGit(pr.base.sha)
+  } else if (github.context.eventName === 'push') {
+    return getChangedFilesFromPush()
+  } else {
+    throw new Error('This action can be triggered only by pull_request or push event')
+  }
+}
+
+async function getChangedFilesFromPush(): Promise<string[] | null> {
+  const push = github.context.payload as Webhooks.WebhookPayloadPush
+
+  // No change detection for pushed tags
+  if (git.isTagRef(push.ref)) return null
+
+  // Get base from input or use repo default branch.
+  // It it starts with 'refs/', it will be trimmed (git fetch refs/heads/<NAME> doesn't work)
+  const baseInput = git.trimRefs(core.getInput('base', {required: false}) || push.repository.default_branch)
+
+  // If base references same branch it was pushed to, we will do comparison against the previously pushed commit.
+  // Otherwise changes are detected against the base reference
+  const base = git.trimRefsHeads(baseInput) === git.trimRefsHeads(push.ref) ? push.before : baseInput
+
+  // There is no previous commit for comparison
+  // e.g. change detection against previous commit of just pushed new branch
+  if (base === git.NULL_SHA) return null
+
+  return await getChangedFilesFromGit(base)
+}
+
+// Fetch base branch and use `git diff` to determine changed files
+async function getChangedFilesFromGit(ref: string): Promise<string[]> {
+  core.debug('Fetching base branch and using `git diff-index` to determine changed files')
+  await git.fetchCommit(ref)
+  // FETCH_HEAD will always point to the just fetched commit
+  // No matter if ref is SHA, branch or tag name or full git ref
+  return await git.getChangedFiles(git.FETCH_HEAD)
+}
+
 // Uses github REST api to get list of files changed in PR
-async function getChangedFiles(
-  client: github.GitHub,
+async function getChangedFilesFromApi(
+  token: string,
   pullRequest: Webhooks.WebhookPayloadPullRequestPullRequest
 ): Promise<string[]> {
+  core.debug('Fetching list of modified files from Github API')
+  const client = new github.GitHub(token)
   const pageSize = 100
   const files: string[] = []
   for (let page = 0; page * pageSize < pullRequest.changed_files; page++) {