switch hashFiles to extension function (#362)

This commit is contained in:
eric sciple
2020-03-18 12:08:51 -04:00
committed by GitHub
parent b0a71481f0
commit dfaf6e06ee
36 changed files with 826 additions and 659 deletions

View File

@@ -24,6 +24,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public const String FetchDepth = "fetch-depth";
public const String GeneratedId = "generated-id";
public const String GitHub = "github";
public const String HashFiles = "hashFiles";
public const String Id = "id";
public const String If = "if";
public const String Image = "image";
@@ -31,6 +32,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public const String Inputs = "inputs";
public const String Job = "job";
public const String JobDefaultsRun = "job-defaults-run";
public const String JobIfResult = "job-if-result";
public const String JobOutputs = "job-outputs";
public const String Jobs = "jobs";
public const String Labels = "labels";
@@ -60,6 +62,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public const String Shell = "shell";
public const String Skipped = "skipped";
public const String StepEnv = "step-env";
public const String StepIfResult = "step-if-result";
public const String Steps = "steps";
public const String StepsScopeInputs = "steps-scope-inputs";
public const String StepsScopeOutputs = "steps-scope-outputs";

View File

@@ -16,6 +16,20 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
{
internal static class PipelineTemplateConverter
{
internal static Boolean ConvertToIfResult(
TemplateContext context,
TemplateToken ifResult)
{
var expression = ifResult.Traverse().FirstOrDefault(x => x is ExpressionToken);
if (expression != null)
{
throw new ArgumentException($"Unexpected type '{expression.GetType().Name}' encountered while reading 'if'.");
}
var evaluationResult = EvaluationResult.CreateIntermediateResult(null, ifResult);
return evaluationResult.IsTruthy;
}
internal static Boolean? ConvertToStepContinueOnError(
TemplateContext context,
TemplateToken token,

View File

@@ -4,7 +4,7 @@ using System.ComponentModel;
using System.Linq;
using System.Threading;
using GitHub.DistributedTask.Expressions2;
using GitHub.DistributedTask.Expressions2.Sdk;
using GitHub.DistributedTask.Expressions2.Sdk.Functions;
using GitHub.DistributedTask.ObjectTemplating;
using GitHub.DistributedTask.ObjectTemplating.Schema;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
@@ -14,6 +14,9 @@ using ITraceWriter = GitHub.DistributedTask.ObjectTemplating.ITraceWriter;
namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
{
/// <summary>
/// Evaluates parts of the workflow DOM. For example, a job strategy or step inputs.
/// </summary>
[EditorBrowsable(EditorBrowsableState.Never)]
public class PipelineTemplateEvaluator
{
@@ -50,13 +53,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public DictionaryContextData EvaluateStepScopeInputs(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(DictionaryContextData);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepsScopeInputs, token, 0, null, omitHeader: true);
@@ -76,13 +80,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public DictionaryContextData EvaluateStepScopeOutputs(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(DictionaryContextData);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepsScopeOutputs, token, 0, null, omitHeader: true);
@@ -102,13 +107,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public Boolean EvaluateStepContinueOnError(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(Boolean?);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.BooleanStepsContext, token, 0, null, omitHeader: true);
@@ -126,16 +132,44 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
return result ?? false;
}
public String EvaluateStepDisplayName(
TemplateToken token,
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(String);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StringStepsContext, token, 0, null, omitHeader: true);
context.Errors.Check();
result = PipelineTemplateConverter.ConvertToStepDisplayName(context, token);
}
catch (Exception ex) when (!(ex is TemplateValidationException))
{
context.Errors.Add(ex);
}
context.Errors.Check();
}
return result;
}
public Dictionary<String, String> EvaluateStepEnvironment(
TemplateToken token,
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions,
StringComparer keyComparer)
{
var result = default(Dictionary<String, String>);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepEnv, token, 0, null, omitHeader: true);
@@ -153,15 +187,44 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
return result ?? new Dictionary<String, String>(keyComparer);
}
public Boolean EvaluateStepIf(
TemplateToken token,
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions,
IEnumerable<KeyValuePair<String, Object>> expressionState)
{
var result = default(Boolean?);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData, expressionFunctions, expressionState);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepIfResult, token, 0, null, omitHeader: true);
context.Errors.Check();
result = PipelineTemplateConverter.ConvertToIfResult(context, token);
}
catch (Exception ex) when (!(ex is TemplateValidationException))
{
context.Errors.Add(ex);
}
context.Errors.Check();
}
return result ?? throw new InvalidOperationException("Step if cannot be null");
}
public Dictionary<String, String> EvaluateStepInputs(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(Dictionary<String, String>);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepWith, token, 0, null, omitHeader: true);
@@ -181,13 +244,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public Int32 EvaluateStepTimeout(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(Int32?);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.NumberStepsContext, token, 0, null, omitHeader: true);
@@ -207,13 +271,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public JobContainer EvaluateJobContainer(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(JobContainer);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.Container, token, 0, null, omitHeader: true);
@@ -233,13 +298,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public Dictionary<String, String> EvaluateJobOutput(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(Dictionary<String, String>);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.JobOutputs, token, 0, null, omitHeader: true);
@@ -269,13 +335,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public Dictionary<String, String> EvaluateJobDefaultsRun(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(Dictionary<String, String>);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.JobDefaultsRun, token, 0, null, omitHeader: true);
@@ -305,13 +372,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public IList<KeyValuePair<String, JobContainer>> EvaluateJobServiceContainers(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(List<KeyValuePair<String, JobContainer>>);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.Services, token, 0, null, omitHeader: true);
@@ -329,62 +397,10 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
return result;
}
public Boolean TryEvaluateStepDisplayName(
TemplateToken token,
private TemplateContext CreateContext(
DictionaryContextData contextData,
out String stepName)
{
stepName = default(String);
var context = CreateContext(contextData);
if (token != null && token.Type != TokenType.Null)
{
// We should only evaluate basic expressions if we are sure we have context on all the Named Values and functions
// Otherwise return and use a default name
if (token is BasicExpressionToken expressionToken)
{
ExpressionNode root = null;
try
{
root = new ExpressionParser().ValidateSyntax(expressionToken.Expression, null) as ExpressionNode;
}
catch (Exception exception)
{
context.Errors.Add(exception);
context.Errors.Check();
}
foreach (var node in root.Traverse())
{
if (node is NamedValue namedValue && !contextData.ContainsKey(namedValue.Name))
{
return false;
}
else if (node is Function function &&
!context.ExpressionFunctions.Any(item => String.Equals(item.Name, function.Name)) &&
!ExpressionConstants.WellKnownFunctions.ContainsKey(function.Name))
{
return false;
}
}
}
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StringStepsContext, token, 0, null, omitHeader: true);
context.Errors.Check();
stepName = PipelineTemplateConverter.ConvertToStepDisplayName(context, token);
}
catch (Exception ex) when (!(ex is TemplateValidationException))
{
context.Errors.Add(ex);
}
context.Errors.Check();
}
return true;
}
private TemplateContext CreateContext(DictionaryContextData contextData)
IList<IFunctionInfo> expressionFunctions,
IEnumerable<KeyValuePair<String, Object>> expressionState = null)
{
var result = new TemplateContext
{
@@ -407,7 +423,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
}
}
// Add named context
// Add named values
if (contextData != null)
{
foreach (var pair in contextData)
@@ -416,14 +432,46 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
}
}
// Compat for new agent against old server
foreach (var name in s_contextNames)
// Add functions
var functionNames = new HashSet<String>(StringComparer.OrdinalIgnoreCase);
if (expressionFunctions?.Count > 0)
{
foreach (var function in expressionFunctions)
{
result.ExpressionFunctions.Add(function);
functionNames.Add(function.Name);
}
}
// Add missing expression values and expression functions.
// This solves the following problems:
// - Compat for new agent against old server (new contexts not sent down in job message)
// - Evaluating early when all referenced contexts are available, even though all allowed
// contexts may not yet be available. For example, evaluating step display name can often
// be performed early.
foreach (var name in s_expressionValueNames)
{
if (!result.ExpressionValues.ContainsKey(name))
{
result.ExpressionValues[name] = null;
}
}
foreach (var name in s_expressionFunctionNames)
{
if (!functionNames.Contains(name))
{
result.ExpressionFunctions.Add(new FunctionInfo<NoOperation>(name, 0, Int32.MaxValue));
}
}
// Add state
if (expressionState != null)
{
foreach (var pair in expressionState)
{
result.State[pair.Key] = pair.Value;
}
}
return result;
}
@@ -431,9 +479,10 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
private readonly ITraceWriter m_trace;
private readonly TemplateSchema m_schema;
private readonly IList<String> m_fileTable;
private readonly String[] s_contextNames = new[]
private readonly String[] s_expressionValueNames = new[]
{
PipelineTemplateConstants.GitHub,
PipelineTemplateConstants.Needs,
PipelineTemplateConstants.Strategy,
PipelineTemplateConstants.Matrix,
PipelineTemplateConstants.Needs,
@@ -444,5 +493,13 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
PipelineTemplateConstants.Runner,
PipelineTemplateConstants.Env,
};
private readonly String[] s_expressionFunctionNames = new[]
{
PipelineTemplateConstants.Always,
PipelineTemplateConstants.Cancelled,
PipelineTemplateConstants.Failure,
PipelineTemplateConstants.HashFiles,
PipelineTemplateConstants.Success,
};
}
}

View File

@@ -2,25 +2,35 @@
using System.ComponentModel;
using System.IO;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using GitHub.DistributedTask.ObjectTemplating.Schema;
namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
{
[EditorBrowsable(EditorBrowsableState.Never)]
public sealed class PipelineTemplateSchemaFactory
public static class PipelineTemplateSchemaFactory
{
public TemplateSchema CreateSchema()
public static TemplateSchema GetSchema()
{
var assembly = Assembly.GetExecutingAssembly();
var json = default(String);
using (var stream = assembly.GetManifestResourceStream("GitHub.DistributedTask.Pipelines.ObjectTemplating.workflow-v1.0.json"))
using (var streamReader = new StreamReader(stream))
if (s_schema == null)
{
json = streamReader.ReadToEnd();
var assembly = Assembly.GetExecutingAssembly();
var json = default(String);
using (var stream = assembly.GetManifestResourceStream("GitHub.DistributedTask.Pipelines.ObjectTemplating.workflow-v1.0.json"))
using (var streamReader = new StreamReader(stream))
{
json = streamReader.ReadToEnd();
}
var objectReader = new JsonObjectReader(null, json);
var schema = TemplateSchema.Load(objectReader);
Interlocked.CompareExchange(ref s_schema, schema, null);
}
var objectReader = new JsonObjectReader(null, json);
return TemplateSchema.Load(objectReader);
return s_schema;
}
private static TemplateSchema s_schema;
}
}

View File

@@ -38,8 +38,8 @@
"steps-scope-input-value": {
"context": [
"github",
"strategy",
"needs",
"strategy",
"matrix",
"secrets",
"steps",
@@ -66,9 +66,9 @@
"steps-scope-output-value": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
@@ -91,9 +91,9 @@
"description": "Default input values for a steps template",
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"one-of": [
"string",
@@ -114,9 +114,9 @@
"description": "Output values for a steps template",
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
@@ -204,6 +204,25 @@
"string": {}
},
"job-if-result": {
"context": [
"github",
"needs",
"always(0,0)",
"failure(0,MAX)",
"cancelled(0,0)",
"success(0,MAX)"
],
"one-of": [
"null",
"boolean",
"number",
"string",
"sequence",
"mapping"
]
},
"strategy": {
"context": [
"github",
@@ -272,9 +291,9 @@
"runs-on": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"one-of": [
"non-empty-string",
@@ -297,10 +316,10 @@
"job-env": {
"context": [
"github",
"secrets",
"needs",
"strategy",
"matrix",
"needs"
"secrets"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -444,9 +463,9 @@
"step-if": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"steps",
"job",
"runner",
@@ -454,7 +473,8 @@
"always(0,0)",
"failure(0,0)",
"cancelled(0,0)",
"success(0,0)"
"success(0,0)",
"hashFiles(1,255)"
],
"string": {}
},
@@ -462,9 +482,9 @@
"step-if-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"steps",
"inputs",
"job",
@@ -473,11 +493,63 @@
"always(0,0)",
"failure(0,0)",
"cancelled(0,0)",
"success(0,0)"
"success(0,0)",
"hashFiles(1,255)"
],
"string": {}
},
"step-if-result": {
"context": [
"github",
"strategy",
"matrix",
"steps",
"job",
"runner",
"env",
"always(0,0)",
"failure(0,0)",
"cancelled(0,0)",
"success(0,0)",
"hashFiles(1,255)"
],
"one-of": [
"null",
"boolean",
"number",
"string",
"sequence",
"mapping"
]
},
"step-if-result-in-template": {
"context": [
"github",
"strategy",
"matrix",
"steps",
"inputs",
"job",
"runner",
"env",
"always(0,0)",
"failure(0,0)",
"cancelled(0,0)",
"success(0,0)",
"hashFiles(1,255)"
],
"one-of": [
"null",
"boolean",
"number",
"string",
"sequence",
"mapping"
]
},
"steps-template-reference": {
"mapping": {
"properties": {
@@ -501,9 +573,9 @@
"steps-template-reference-inputs": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
@@ -519,9 +591,9 @@
"steps-template-reference-inputs-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
@@ -538,14 +610,15 @@
"step-env": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -556,15 +629,16 @@
"step-env-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -575,14 +649,35 @@
"step-with": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"mapping": {
"loose-key-type": "non-empty-string",
"loose-value-type": "string"
}
},
"step-with-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"secrets",
"steps",
"inputs",
"job",
"runner",
"env",
"hashFiles(1,255)"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -593,9 +688,9 @@
"container": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"one-of": [
"string",
@@ -618,9 +713,9 @@
"services": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -631,9 +726,9 @@
"services-container": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"one-of": [
"non-empty-string",
@@ -648,25 +743,6 @@
}
},
"step-with-in-template": {
"context": [
"github",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
"job",
"runner",
"env"
],
"mapping": {
"loose-key-type": "non-empty-string",
"loose-value-type": "string"
}
},
"non-empty-string": {
"string": {
"require-non-empty": true
@@ -682,9 +758,9 @@
"boolean-strategy-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"boolean": {}
},
@@ -692,9 +768,9 @@
"number-strategy-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"number": {}
},
@@ -702,9 +778,9 @@
"string-strategy-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"string": {}
},
@@ -712,14 +788,15 @@
"boolean-steps-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"boolean": {}
},
@@ -727,15 +804,16 @@
"boolean-steps-context-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"boolean": {}
},
@@ -743,14 +821,15 @@
"number-steps-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"number": {}
},
@@ -758,15 +837,16 @@
"number-steps-context-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"number": {}
},
@@ -774,9 +854,9 @@
"string-runner-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
@@ -789,14 +869,15 @@
"string-steps-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"string": {}
},
@@ -804,15 +885,16 @@
"string-steps-context-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"string": {}
}