change hashFiles() expression to use @actions/glob. (#268)

.gitignore

@@ -2,6 +2,7 @@
 **/bin
 **/obj
 **/libs
+**/lib
 
 # editors
 **/*.xproj

src/Misc/expressionFunc/hashFiles/.eslintignore

@@ -0,0 +1,3 @@
+dist/
+lib/
+node_modules/

src/Misc/expressionFunc/hashFiles/.eslintrc.json

@@ -0,0 +1,59 @@
+{
+    "plugins": ["jest", "@typescript-eslint"],
+    "extends": ["plugin:github/es6"],
+    "parser": "@typescript-eslint/parser",
+    "parserOptions": {
+      "ecmaVersion": 9,
+      "sourceType": "module",
+      "project": "./tsconfig.json"
+    },
+    "rules": {
+      "eslint-comments/no-use": "off",
+      "import/no-namespace": "off",
+      "no-console": "off",
+      "no-unused-vars": "off",
+      "@typescript-eslint/no-unused-vars": "error",
+      "@typescript-eslint/explicit-member-accessibility": ["error", {"accessibility": "no-public"}],
+      "@typescript-eslint/no-require-imports": "error",
+      "@typescript-eslint/array-type": "error",
+      "@typescript-eslint/await-thenable": "error",
+      "@typescript-eslint/ban-ts-ignore": "error",
+      "camelcase": "off",
+      "@typescript-eslint/camelcase": "error",
+      "@typescript-eslint/class-name-casing": "error",
+      "@typescript-eslint/explicit-function-return-type": ["error", {"allowExpressions": true}],
+      "@typescript-eslint/func-call-spacing": ["error", "never"],
+      "@typescript-eslint/generic-type-naming": ["error", "^[A-Z][A-Za-z]*$"],
+      "@typescript-eslint/no-array-constructor": "error",
+      "@typescript-eslint/no-empty-interface": "error",
+      "@typescript-eslint/no-explicit-any": "error",
+      "@typescript-eslint/no-extraneous-class": "error",
+      "@typescript-eslint/no-for-in-array": "error",
+      "@typescript-eslint/no-inferrable-types": "error",
+      "@typescript-eslint/no-misused-new": "error",
+      "@typescript-eslint/no-namespace": "error",
+      "@typescript-eslint/no-non-null-assertion": "warn",
+      "@typescript-eslint/no-object-literal-type-assertion": "error",
+      "@typescript-eslint/no-unnecessary-qualifier": "error",
+      "@typescript-eslint/no-unnecessary-type-assertion": "error",
+      "@typescript-eslint/no-useless-constructor": "error",
+      "@typescript-eslint/no-var-requires": "error",
+      "@typescript-eslint/prefer-for-of": "warn",
+      "@typescript-eslint/prefer-function-type": "warn",
+      "@typescript-eslint/prefer-includes": "error",
+      "@typescript-eslint/prefer-interface": "error",
+      "@typescript-eslint/prefer-string-starts-ends-with": "error",
+      "@typescript-eslint/promise-function-async": "error",
+      "@typescript-eslint/require-array-sort-compare": "error",
+      "@typescript-eslint/restrict-plus-operands": "error",
+      "semi": "off",
+      "@typescript-eslint/semi": ["error", "never"],
+      "@typescript-eslint/type-annotation-spacing": "error",
+      "@typescript-eslint/unbound-method": "error"
+    },
+    "env": {
+      "node": true,
+      "es6": true,
+      "jest/globals": true
+    }
+  }

src/Misc/expressionFunc/hashFiles/.prettierignore

@@ -0,0 +1,3 @@
+dist/
+lib/
+node_modules/

src/Misc/expressionFunc/hashFiles/.prettierrc.json

@@ -0,0 +1,11 @@
+{
+    "printWidth": 80,
+    "tabWidth": 2,
+    "useTabs": false,
+    "semi": false,
+    "singleQuote": true,
+    "trailingComma": "none",
+    "bracketSpacing": false,
+    "arrowParens": "avoid",
+    "parser": "typescript"
+  }

src/Misc/expressionFunc/hashFiles/README.md

@@ -0,0 +1 @@
+To update hashFiles under `Misc/layoutbin` run `npm install && npm run all`

src/Misc/expressionFunc/hashFiles/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "hashFiles",
+  "version": "1.0.0",
+  "description": "GitHub Actions HashFiles() expression function",
+  "main": "lib/hashFiles.js",
+  "scripts": {
+    "build": "tsc",
+    "format": "prettier --write **/*.ts",
+    "format-check": "prettier --check **/*.ts",
+    "lint": "eslint src/**/*.ts",
+    "pack": "ncc build -o ../../layoutbin/hashFiles",
+    "all": "npm run build && npm run format && npm run lint && npm run pack"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/actions/runner.git"
+  },
+  "keywords": [
+    "actions"
+  ],
+  "author": "GitHub Actions",
+  "license": "MIT",
+  "dependencies": {
+    "@actions/glob": "^0.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^12.7.12",
+    "@typescript-eslint/parser": "^2.8.0",
+    "@zeit/ncc": "^0.20.5",
+    "eslint": "^5.16.0",
+    "eslint-plugin-github": "^2.0.0",
+    "prettier": "^1.19.1",
+    "typescript": "^3.6.4"
+  }
+}

src/Misc/expressionFunc/hashFiles/src/hashFiles.ts

@@ -0,0 +1,55 @@
+import * as glob from '@actions/glob'
+import * as crypto from 'crypto'
+import * as fs from 'fs'
+import * as stream from 'stream'
+import * as util from 'util'
+import * as path from 'path'
+
+async function run(): Promise<void> {
+  // arg0 -> node
+  // arg1 -> hashFiles.js
+  // env[followSymbolicLinks] = true/null
+  // env[patterns] -> glob patterns
+  let followSymbolicLinks = false
+  const matchPatterns = process.env.patterns || ''
+  if (process.env.followSymbolicLinks === 'true') {
+    console.log('Follow symbolic links')
+    followSymbolicLinks = true
+  }
+
+  console.log(`Match Pattern: ${matchPatterns}`)
+  let hasMatch = false
+  const githubWorkspace = process.cwd()
+  const result = crypto.createHash('sha256')
+  let count = 0
+  const globber = await glob.create(matchPatterns, {followSymbolicLinks})
+  for await (const file of globber.globGenerator()) {
+    console.log(file)
+    if (!file.startsWith(`${githubWorkspace}${path.sep}`)) {
+      console.log(`Ignore '${file}' since it is not under GITHUB_WORKSPACE.`)
+      continue
+    }
+    if (fs.statSync(file).isDirectory()) {
+      console.log(`Skip directory '${file}'.`)
+      continue
+    }
+    const hash = crypto.createHash('sha256')
+    const pipeline = util.promisify(stream.pipeline)
+    await pipeline(fs.createReadStream(file), hash)
+    result.write(hash.digest())
+    count++
+    if (!hasMatch) {
+      hasMatch = true
+    }
+  }
+  result.end()
+
+  if (hasMatch) {
+    console.log(`Find ${count} files to hash.`)
+    console.error(`__OUTPUT__${result.digest('hex')}__OUTPUT__`)
+  } else {
+    console.error(`__OUTPUT____OUTPUT__`)
+  }
+}
+
+run()

src/Misc/expressionFunc/hashFiles/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "compilerOptions": {
+    "target": "es6",                          /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */
+    "module": "commonjs",                     /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
+    "outDir": "./lib",                        /* Redirect output structure to the directory. */
+    "rootDir": "./src",                       /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
+    "strict": true,                           /* Enable all strict type-checking options. */
+    "noImplicitAny": true,                    /* Raise error on expressions and declarations with an implied 'any' type. */
+    "esModuleInterop": true                   /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
+  },
+  "exclude": ["node_modules", "**/*.test.ts"]
+}

src/Runner.Worker/ExecutionContext.cs

@@ -20,6 +20,7 @@ using System.Text;
 using System.Collections;
 using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating;
 using Pipelines = GitHub.DistributedTask.Pipelines;
+using GitHub.DistributedTask.Expressions2;
 
 namespace GitHub.Runner.Worker
 {
@@ -568,6 +569,12 @@ namespace GitHub.Runner.Worker
                 }
             }
 
+            // Expression functions
+            if (Variables.GetBoolean("System.HashFilesV2") == true)
+            {
+                ExpressionConstants.UpdateFunction<Handlers.HashFiles>("hashFiles", 1, byte.MaxValue);
+            }
+
             // Expression values
             if (message.ContextData?.Count > 0)
             {

src/Runner.Worker/ExpressionFunctions/HashFiles.cs

@@ -0,0 +1,126 @@
+using System;
+using System.IO;
+using GitHub.DistributedTask.Expressions2.Sdk;
+using GitHub.DistributedTask.Pipelines.ContextData;
+using GitHub.DistributedTask.Pipelines.ObjectTemplating;
+using GitHub.Runner.Sdk;
+using System.Reflection;
+using System.Threading;
+using System.Collections.Generic;
+
+namespace GitHub.Runner.Worker.Handlers
+{
+    public class FunctionTrace : ITraceWriter
+    {
+        private GitHub.DistributedTask.Expressions2.ITraceWriter _trace;
+
+        public FunctionTrace(GitHub.DistributedTask.Expressions2.ITraceWriter trace)
+        {
+            _trace = trace;
+        }
+        public void Info(string message)
+        {
+            _trace.Info(message);
+        }
+
+        public void Verbose(string message)
+        {
+            _trace.Info(message);
+        }
+    }
+
+    public sealed class HashFiles : Function
+    {
+        protected sealed override Object EvaluateCore(
+            EvaluationContext context,
+            out ResultMemory resultMemory)
+        {
+            resultMemory = null;
+            var templateContext = context.State as DistributedTask.ObjectTemplating.TemplateContext;
+            ArgUtil.NotNull(templateContext, nameof(templateContext));
+            templateContext.ExpressionValues.TryGetValue(PipelineTemplateConstants.GitHub, out var githubContextData);
+            ArgUtil.NotNull(githubContextData, nameof(githubContextData));
+            var githubContext = githubContextData as DictionaryContextData;
+            ArgUtil.NotNull(githubContext, nameof(githubContext));
+            githubContext.TryGetValue(PipelineTemplateConstants.Workspace, out var workspace);
+            var workspaceData = workspace as StringContextData;
+            ArgUtil.NotNull(workspaceData, nameof(workspaceData));
+
+            string githubWorkspace = workspaceData.Value;
+            bool followSymlink = false;
+            List<string> patterns = new List<string>();
+            var firstParameter = true;
+            foreach (var parameter in Parameters)
+            {
+                var parameterString = parameter.Evaluate(context).ConvertToString();
+                if (firstParameter)
+                {
+                    firstParameter = false;
+                    if (parameterString.StartsWith("--"))
+                    {
+                        if (string.Equals(parameterString, "--follow-symbolic-links", StringComparison.OrdinalIgnoreCase))
+                        {
+                            followSymlink = true;
+                            continue;
+                        }
+                        else
+                        {
+                            throw new ArgumentOutOfRangeException($"Invalid glob option {parameterString}, avaliable option: '--follow-symbolic-links'.");
+                        }
+                    }
+                }
+
+                patterns.Add(parameterString);
+            }
+
+            context.Trace.Info($"Search root directory: '{githubWorkspace}'");
+            context.Trace.Info($"Search pattern: '{string.Join(", ", patterns)}'");
+
+            string binDir = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
+            string runnerRoot = new DirectoryInfo(binDir).Parent.FullName;
+
+            string node = Path.Combine(runnerRoot, "externals", "node12", "bin", $"node{IOUtil.ExeExtension}");
+            string hashFilesScript = Path.Combine(binDir, "hashFiles");
+            var hashResult = string.Empty;
+            var p = new ProcessInvoker(new FunctionTrace(context.Trace));
+            p.ErrorDataReceived += ((_, data) =>
+            {
+                if (!string.IsNullOrEmpty(data.Data) && data.Data.StartsWith("__OUTPUT__") && data.Data.EndsWith("__OUTPUT__"))
+                {
+                    hashResult = data.Data.Substring(10, data.Data.Length - 20);
+                    context.Trace.Info($"Hash result: '{hashResult}'");
+                }
+                else
+                {
+                    context.Trace.Info(data.Data);
+                }
+            });
+
+            p.OutputDataReceived += ((_, data) =>
+            {
+                context.Trace.Info(data.Data);
+            });
+
+            var env = new Dictionary<string, string>();
+            if (followSymlink)
+            {
+                env["followSymbolicLinks"] = "true";
+            }
+            env["patterns"] = string.Join(Environment.NewLine, patterns);
+
+            int exitCode = p.ExecuteAsync(workingDirectory: githubWorkspace,
+                                          fileName: node,
+                                          arguments: $"\"{hashFilesScript.Replace("\"", "\\\"")}\"",
+                                          environment: env,
+                                          requireExitCodeZero: false,
+                                          cancellationToken: new CancellationTokenSource(TimeSpan.FromSeconds(120)).Token).GetAwaiter().GetResult();
+
+            if (exitCode != 0)
+            {
+                throw new InvalidOperationException($"hashFiles('{ExpressionUtility.StringEscape(string.Join(", ", patterns))}') failed. Fail to hash files under directory '{githubWorkspace}'");
+            }
+
+            return hashResult;
+        }
+    }
+}

src/Sdk/DTExpressions2/Expressions2/ExpressionConstants.cs

@@ -5,7 +5,7 @@ using GitHub.DistributedTask.Expressions2.Sdk.Functions;
 
 namespace GitHub.DistributedTask.Expressions2
 {
-    internal static class ExpressionConstants
+    public static class ExpressionConstants
     {
         static ExpressionConstants()
         {
@@ -24,6 +24,12 @@ namespace GitHub.DistributedTask.Expressions2
             WellKnownFunctions.Add(name, new FunctionInfo<T>(name, minParameters, maxParameters));
         }
 
+        public static void UpdateFunction<T>(String name, Int32 minParameters, Int32 maxParameters)
+                    where T : Function, new()
+        {
+            WellKnownFunctions[name] = new FunctionInfo<T>(name, minParameters, maxParameters);
+        }
+
         internal static readonly String False = "false";
         internal static readonly String Infinity = "Infinity";
         internal static readonly Int32 MaxDepth = 50;