switch hashFiles to extension function (#362)

2025-12-13 10:05:23 +00:00 · 2020-03-18 12:08:51 -04:00
parent b0a71481f0
commit dfaf6e06ee
36 changed files with 826 additions and 659 deletions
--- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs
+++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs
@@ -24,6 +24,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
        public const String FetchDepth = "fetch-depth";
        public const String GeneratedId = "generated-id";
        public const String GitHub = "github";
+        public const String HashFiles = "hashFiles";
        public const String Id = "id";
        public const String If = "if";
        public const String Image = "image";
@@ -31,6 +32,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
        public const String Inputs = "inputs";
        public const String Job = "job";
        public const String JobDefaultsRun = "job-defaults-run";
+        public const String JobIfResult = "job-if-result";
        public const String JobOutputs = "job-outputs";
        public const String Jobs = "jobs";
        public const String Labels = "labels";
@@ -60,6 +62,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
        public const String Shell = "shell";
        public const String Skipped = "skipped";
        public const String StepEnv = "step-env";
+        public const String StepIfResult = "step-if-result";
        public const String Steps = "steps";
        public const String StepsScopeInputs = "steps-scope-inputs";
        public const String StepsScopeOutputs = "steps-scope-outputs";
--- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs
+++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs
@@ -16,6 +16,20 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
 {
    internal static class PipelineTemplateConverter
    {
+        internal static Boolean ConvertToIfResult(
+            TemplateContext context,
+            TemplateToken ifResult)
+        {
+            var expression = ifResult.Traverse().FirstOrDefault(x => x is ExpressionToken);
+            if (expression != null)
+            {
+                throw new ArgumentException($"Unexpected type '{expression.GetType().Name}' encountered while reading 'if'.");
+            }
+
+            var evaluationResult = EvaluationResult.CreateIntermediateResult(null, ifResult);
+            return evaluationResult.IsTruthy;
+        }
+
        internal static Boolean? ConvertToStepContinueOnError(
            TemplateContext context,
            TemplateToken token,
--- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs
+++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs
@@ -4,7 +4,7 @@ using System.ComponentModel;
 using System.Linq;
 using System.Threading;
 using GitHub.DistributedTask.Expressions2;
-using GitHub.DistributedTask.Expressions2.Sdk;
+using GitHub.DistributedTask.Expressions2.Sdk.Functions;
 using GitHub.DistributedTask.ObjectTemplating;
 using GitHub.DistributedTask.ObjectTemplating.Schema;
 using GitHub.DistributedTask.ObjectTemplating.Tokens;
@@ -14,6 +14,9 @@ using ITraceWriter = GitHub.DistributedTask.ObjectTemplating.ITraceWriter;

 namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
 {
+    /// <summary>
+    /// Evaluates parts of the workflow DOM. For example, a job strategy or step inputs.
+    /// </summary>
    [EditorBrowsable(EditorBrowsableState.Never)]
    public class PipelineTemplateEvaluator
    {
@@ -50,13 +53,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating

        public DictionaryContextData EvaluateStepScopeInputs(
            TemplateToken token,
-            DictionaryContextData contextData)
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
        {
            var result = default(DictionaryContextData);

            if (token != null && token.Type != TokenType.Null)
            {
-                var context = CreateContext(contextData);
+                var context = CreateContext(contextData, expressionFunctions);
                try
                {
                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepsScopeInputs, token, 0, null, omitHeader: true);
@@ -76,13 +80,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating

        public DictionaryContextData EvaluateStepScopeOutputs(
            TemplateToken token,
-            DictionaryContextData contextData)
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
        {
            var result = default(DictionaryContextData);

            if (token != null && token.Type != TokenType.Null)
            {
-                var context = CreateContext(contextData);
+                var context = CreateContext(contextData, expressionFunctions);
                try
                {
                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepsScopeOutputs, token, 0, null, omitHeader: true);
@@ -102,13 +107,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating

        public Boolean EvaluateStepContinueOnError(
            TemplateToken token,
-            DictionaryContextData contextData)
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
        {
            var result = default(Boolean?);

            if (token != null && token.Type != TokenType.Null)
            {
-                var context = CreateContext(contextData);
+                var context = CreateContext(contextData, expressionFunctions);
                try
                {
                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.BooleanStepsContext, token, 0, null, omitHeader: true);
@@ -126,16 +132,44 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
            return result ?? false;
        }

+        public String EvaluateStepDisplayName(
+            TemplateToken token,
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
+        {
+            var result = default(String);
+
+            if (token != null && token.Type != TokenType.Null)
+            {
+                var context = CreateContext(contextData, expressionFunctions);
+                try
+                {
+                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StringStepsContext, token, 0, null, omitHeader: true);
+                    context.Errors.Check();
+                    result = PipelineTemplateConverter.ConvertToStepDisplayName(context, token);
+                }
+                catch (Exception ex) when (!(ex is TemplateValidationException))
+                {
+                    context.Errors.Add(ex);
+                }
+
+                context.Errors.Check();
+            }
+
+            return result;
+        }
+
        public Dictionary<String, String> EvaluateStepEnvironment(
            TemplateToken token,
            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions,
            StringComparer keyComparer)
        {
            var result = default(Dictionary<String, String>);

            if (token != null && token.Type != TokenType.Null)
            {
-                var context = CreateContext(contextData);
+                var context = CreateContext(contextData, expressionFunctions);
                try
                {
                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepEnv, token, 0, null, omitHeader: true);
@@ -153,15 +187,44 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
            return result ?? new Dictionary<String, String>(keyComparer);
        }

+        public Boolean EvaluateStepIf(
+            TemplateToken token,
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions,
+            IEnumerable<KeyValuePair<String, Object>> expressionState)
+        {
+            var result = default(Boolean?);
+
+            if (token != null && token.Type != TokenType.Null)
+            {
+                var context = CreateContext(contextData, expressionFunctions, expressionState);
+                try
+                {
+                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepIfResult, token, 0, null, omitHeader: true);
+                    context.Errors.Check();
+                    result = PipelineTemplateConverter.ConvertToIfResult(context, token);
+                }
+                catch (Exception ex) when (!(ex is TemplateValidationException))
+                {
+                    context.Errors.Add(ex);
+                }
+
+                context.Errors.Check();
+            }
+
+            return result ?? throw new InvalidOperationException("Step if cannot be null");
+        }
+
        public Dictionary<String, String> EvaluateStepInputs(
            TemplateToken token,
-            DictionaryContextData contextData)
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
        {
            var result = default(Dictionary<String, String>);

            if (token != null && token.Type != TokenType.Null)
            {
-                var context = CreateContext(contextData);
+                var context = CreateContext(contextData, expressionFunctions);
                try
                {
                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepWith, token, 0, null, omitHeader: true);
@@ -181,13 +244,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating

        public Int32 EvaluateStepTimeout(
            TemplateToken token,
-            DictionaryContextData contextData)
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
        {
            var result = default(Int32?);

            if (token != null && token.Type != TokenType.Null)
            {
-                var context = CreateContext(contextData);
+                var context = CreateContext(contextData, expressionFunctions);
                try
                {
                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.NumberStepsContext, token, 0, null, omitHeader: true);
@@ -207,13 +271,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating

        public JobContainer EvaluateJobContainer(
            TemplateToken token,
-            DictionaryContextData contextData)
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
        {
            var result = default(JobContainer);

            if (token != null && token.Type != TokenType.Null)
            {
-                var context = CreateContext(contextData);
+                var context = CreateContext(contextData, expressionFunctions);
                try
                {
                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.Container, token, 0, null, omitHeader: true);
@@ -233,13 +298,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating

        public Dictionary<String, String> EvaluateJobOutput(
            TemplateToken token,
-            DictionaryContextData contextData)
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
        {
            var result = default(Dictionary<String, String>);

            if (token != null && token.Type != TokenType.Null)
            {
-                var context = CreateContext(contextData);
+                var context = CreateContext(contextData, expressionFunctions);
                try
                {
                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.JobOutputs, token, 0, null, omitHeader: true);
@@ -269,13 +335,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating

        public Dictionary<String, String> EvaluateJobDefaultsRun(
            TemplateToken token,
-            DictionaryContextData contextData)
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
        {
            var result = default(Dictionary<String, String>);

            if (token != null && token.Type != TokenType.Null)
            {
-                var context = CreateContext(contextData);
+                var context = CreateContext(contextData, expressionFunctions);
                try
                {
                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.JobDefaultsRun, token, 0, null, omitHeader: true);
@@ -305,13 +372,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating

        public IList<KeyValuePair<String, JobContainer>> EvaluateJobServiceContainers(
            TemplateToken token,
-            DictionaryContextData contextData)
+            DictionaryContextData contextData,
+            IList<IFunctionInfo> expressionFunctions)
        {
            var result = default(List<KeyValuePair<String, JobContainer>>);

            if (token != null && token.Type != TokenType.Null)
            {
-                var context = CreateContext(contextData);
+                var context = CreateContext(contextData, expressionFunctions);
                try
                {
                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.Services, token, 0, null, omitHeader: true);
@@ -329,62 +397,10 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
            return result;
        }

-        public Boolean TryEvaluateStepDisplayName(
-            TemplateToken token,
+        private TemplateContext CreateContext(
            DictionaryContextData contextData,
-            out String stepName)
-        {
-            stepName = default(String);
-            var context = CreateContext(contextData);
-
-            if (token != null && token.Type != TokenType.Null)
-            {
-                // We should only evaluate basic expressions if we are sure we have context on all the Named Values and functions
-                // Otherwise return and use a default name
-                if (token is BasicExpressionToken expressionToken)
-                {
-                    ExpressionNode root = null;
-                    try
-                    {
-                        root = new ExpressionParser().ValidateSyntax(expressionToken.Expression, null) as ExpressionNode;
-                    }
-                    catch (Exception exception)
-                    {
-                        context.Errors.Add(exception);
-                        context.Errors.Check();
-                    }
-                    foreach (var node in root.Traverse())
-                    {
-                        if (node is NamedValue namedValue && !contextData.ContainsKey(namedValue.Name))
-                        {
-                            return false;
-                        }
-                        else if (node is Function function &&
-                            !context.ExpressionFunctions.Any(item => String.Equals(item.Name, function.Name)) &&
-                            !ExpressionConstants.WellKnownFunctions.ContainsKey(function.Name))
-                        {
-                            return false;
-                        }
-                    }
-                }
-
-                try
-                {
-                    token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StringStepsContext, token, 0, null, omitHeader: true);
-                    context.Errors.Check();
-                    stepName = PipelineTemplateConverter.ConvertToStepDisplayName(context, token);
-                }
-                catch (Exception ex) when (!(ex is TemplateValidationException))
-                {
-                    context.Errors.Add(ex);
-                }
-
-                context.Errors.Check();
-            }
-            return true;
-        }
-
-        private TemplateContext CreateContext(DictionaryContextData contextData)
+            IList<IFunctionInfo> expressionFunctions,
+            IEnumerable<KeyValuePair<String, Object>> expressionState = null)
        {
            var result = new TemplateContext
            {
@@ -407,7 +423,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
                }
            }

-            // Add named context
+            // Add named values
            if (contextData != null)
            {
                foreach (var pair in contextData)
@@ -416,14 +432,46 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
                }
            }

-            // Compat for new agent against old server
-            foreach (var name in s_contextNames)
+            // Add functions
+            var functionNames = new HashSet<String>(StringComparer.OrdinalIgnoreCase);
+            if (expressionFunctions?.Count > 0)
+            {
+                foreach (var function in expressionFunctions)
+                {
+                    result.ExpressionFunctions.Add(function);
+                    functionNames.Add(function.Name);
+                }
+            }
+
+            // Add missing expression values and expression functions.
+            // This solves the following problems:
+            //   - Compat for new agent against old server (new contexts not sent down in job message)
+            //   - Evaluating early when all referenced contexts are available, even though all allowed
+            //     contexts may not yet be available. For example, evaluating step display name can often
+            //     be performed early.
+            foreach (var name in s_expressionValueNames)
            {
                if (!result.ExpressionValues.ContainsKey(name))
                {
                    result.ExpressionValues[name] = null;
                }
            }
+            foreach (var name in s_expressionFunctionNames)
+            {
+                if (!functionNames.Contains(name))
+                {
+                    result.ExpressionFunctions.Add(new FunctionInfo<NoOperation>(name, 0, Int32.MaxValue));
+                }
+            }
+
+            // Add state
+            if (expressionState != null)
+            {
+                foreach (var pair in expressionState)
+                {
+                    result.State[pair.Key] = pair.Value;
+                }
+            }

            return result;
        }
@@ -431,9 +479,10 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
        private readonly ITraceWriter m_trace;
        private readonly TemplateSchema m_schema;
        private readonly IList<String> m_fileTable;
-        private readonly String[] s_contextNames = new[]
+        private readonly String[] s_expressionValueNames = new[]
        {
            PipelineTemplateConstants.GitHub,
+            PipelineTemplateConstants.Needs,
            PipelineTemplateConstants.Strategy,
            PipelineTemplateConstants.Matrix,
            PipelineTemplateConstants.Needs,
@@ -444,5 +493,13 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
            PipelineTemplateConstants.Runner,
            PipelineTemplateConstants.Env,
        };
+        private readonly String[] s_expressionFunctionNames = new[]
+        {
+            PipelineTemplateConstants.Always,
+            PipelineTemplateConstants.Cancelled,
+            PipelineTemplateConstants.Failure,
+            PipelineTemplateConstants.HashFiles,
+            PipelineTemplateConstants.Success,
+        };
    }
 }
--- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateSchemaFactory.cs
+++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateSchemaFactory.cs
@@ -2,25 +2,35 @@
 using System.ComponentModel;
 using System.IO;
 using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
 using GitHub.DistributedTask.ObjectTemplating.Schema;

 namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
 {
    [EditorBrowsable(EditorBrowsableState.Never)]
-    public sealed class PipelineTemplateSchemaFactory
+    public static class PipelineTemplateSchemaFactory
    {
-        public TemplateSchema CreateSchema()
+        public static TemplateSchema GetSchema()
        {
-            var assembly = Assembly.GetExecutingAssembly();
-            var json = default(String);
-            using (var stream = assembly.GetManifestResourceStream("GitHub.DistributedTask.Pipelines.ObjectTemplating.workflow-v1.0.json"))
-            using (var streamReader = new StreamReader(stream))
+            if (s_schema == null)
            {
-                json = streamReader.ReadToEnd();
+                var assembly = Assembly.GetExecutingAssembly();
+                var json = default(String);
+                using (var stream = assembly.GetManifestResourceStream("GitHub.DistributedTask.Pipelines.ObjectTemplating.workflow-v1.0.json"))
+                using (var streamReader = new StreamReader(stream))
+                {
+                    json = streamReader.ReadToEnd();
+                }
+
+                var objectReader = new JsonObjectReader(null, json);
+                var schema = TemplateSchema.Load(objectReader);
+                Interlocked.CompareExchange(ref s_schema, schema, null);
            }

-            var objectReader = new JsonObjectReader(null, json);
-            return TemplateSchema.Load(objectReader);
+            return s_schema;
        }
+
+        private static TemplateSchema s_schema;
    }
 }