actions/runner
1
0
Fork 0
mirror of https://github.com/actions/runner.git synced 2025-12-16 15:06:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

switch hashFiles to extension function (#362)

Browse Source
This commit is contained in:
eric sciple
2020-03-18 12:08:51 -04:00
committed by GitHub
parent b0a71481f0
commit dfaf6e06ee
36 changed files with 826 additions and 659 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/Sdk/DTExpressions2/Expressions2/ExpressionConstants.cs
View File

@@ -5,7 +5,7 @@ using GitHub.DistributedTask.Expressions2.Sdk.Functions;
namespace GitHub.DistributedTask.Expressions2
{
public static class ExpressionConstants
internal static class ExpressionConstants
{
static ExpressionConstants()
{
@@ -16,7 +16,6 @@ namespace GitHub.DistributedTask.Expressions2
AddFunction<StartsWith>("startsWith", 2, 2);
AddFunction<ToJson>("toJson", 1, 1);
AddFunction<FromJson>("fromJson", 1, 1);
AddFunction<HashFiles>("hashFiles", 1, 1);
}
private static void AddFunction<T>(String name, Int32 minParameters, Int32 maxParameters)
@@ -25,12 +24,6 @@ namespace GitHub.DistributedTask.Expressions2
WellKnownFunctions.Add(name, new FunctionInfo<T>(name, minParameters, maxParameters));
}
public static void UpdateFunction<T>(String name, Int32 minParameters, Int32 maxParameters)
where T : Function, new()
{
WellKnownFunctions[name] = new FunctionInfo<T>(name, minParameters, maxParameters);
}
internal static readonly String False = "false";
internal static readonly String Infinity = "Infinity";
internal static readonly Int32 MaxDepth = 50;

122
src/Sdk/DTExpressions2/Expressions2/Sdk/Functions/HashFiles.cs
View File

@@ -1,122 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Minimatch;
using System.IO;
using System.Security.Cryptography;
using GitHub.DistributedTask.Expressions2.Sdk;
using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.Pipelines.ObjectTemplating;
namespace GitHub.DistributedTask.Expressions2.Sdk.Functions
{
internal sealed class HashFiles : Function
{
protected sealed override Object EvaluateCore(
EvaluationContext context,
out ResultMemory resultMemory)
{
resultMemory = null;
// hashFiles() only works on the runner and only works with files under GITHUB_WORKSPACE
// Since GITHUB_WORKSPACE is set by runner, I am using that as the fact of this code runs on server or runner.
if (context.State is ObjectTemplating.TemplateContext templateContext &&
templateContext.ExpressionValues.TryGetValue(PipelineTemplateConstants.GitHub, out var githubContextData) &&
githubContextData is DictionaryContextData githubContext &&
githubContext.TryGetValue(PipelineTemplateConstants.Workspace, out var workspace) == true &&
workspace is StringContextData workspaceData)
{
string searchRoot = workspaceData.Value;
string pattern = Parameters[0].Evaluate(context).ConvertToString();
// Convert slashes on Windows
if (s_isWindows)
{
pattern = pattern.Replace('\\', '/');
}
// Root the pattern
if (!Path.IsPathRooted(pattern))
{
var patternRoot = s_isWindows ? searchRoot.Replace('\\', '/').TrimEnd('/') : searchRoot.TrimEnd('/');
pattern = string.Concat(patternRoot, "/", pattern);
}
// Get all files
context.Trace.Info($"Search root directory: '{searchRoot}'");
context.Trace.Info($"Search pattern: '{pattern}'");
var files = Directory.GetFiles(searchRoot, "*", SearchOption.AllDirectories)
.Select(x => s_isWindows ? x.Replace('\\', '/') : x)
.OrderBy(x => x, StringComparer.Ordinal)
.ToList();
if (files.Count == 0)
{
throw new ArgumentException($"hashFiles('{ExpressionUtility.StringEscape(pattern)}') failed. Directory '{searchRoot}' is empty");
}
else
{
context.Trace.Info($"Found {files.Count} files");
}
// Match
var matcher = new Minimatcher(pattern, s_minimatchOptions);
files = matcher.Filter(files)
.Select(x => s_isWindows ? x.Replace('/', '\\') : x)
.ToList();
if (files.Count == 0)
{
throw new ArgumentException($"hashFiles('{ExpressionUtility.StringEscape(pattern)}') failed. Search pattern '{pattern}' doesn't match any file under '{searchRoot}'");
}
else
{
context.Trace.Info($"{files.Count} matches to hash");
}
// Hash each file
List<byte> filesSha256 = new List<byte>();
foreach (var file in files)
{
context.Trace.Info($"Hash {file}");
using (SHA256 sha256hash = SHA256.Create())
{
using (var fileStream = File.OpenRead(file))
{
filesSha256.AddRange(sha256hash.ComputeHash(fileStream));
}
}
}
// Hash the hashes
using (SHA256 sha256hash = SHA256.Create())
{
var hashBytes = sha256hash.ComputeHash(filesSha256.ToArray());
StringBuilder hashString = new StringBuilder();
for (int i = 0; i < hashBytes.Length; i++)
{
hashString.Append(hashBytes[i].ToString("x2"));
}
var result = hashString.ToString();
context.Trace.Info($"Final hash result: '{result}'");
return result;
}
}
else
{
throw new InvalidOperationException("'hashfiles' expression function is only supported under runner context.");
}
}
private static readonly bool s_isWindows = Environment.OSVersion.Platform != PlatformID.Unix && Environment.OSVersion.Platform != PlatformID.MacOSX;
// Only support basic globbing (* ? and []) and globstar (**)
private static readonly Options s_minimatchOptions = new Options
{
Dot = true,
NoBrace = true,
NoCase = s_isWindows,
NoComment = true,
NoExt = true,
NoNegate = true,
};
}
}

38
src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/Definition.cs
View File

@@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Linq;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
@@ -22,10 +23,27 @@ namespace GitHub.DistributedTask.ObjectTemplating.Schema
{
var context = definition[i].Value.AssertSequence($"{TemplateConstants.Context}");
definition.RemoveAt(i);
Context = context
.Select(x => x.AssertString($"{TemplateConstants.Context} item").Value)
.Distinct()
.ToArray();
var readerContext = new HashSet<String>(StringComparer.OrdinalIgnoreCase);
var evaluatorContext = new HashSet<String>(StringComparer.OrdinalIgnoreCase);
foreach (TemplateToken item in context)
{
var itemStr = item.AssertString($"{TemplateConstants.Context} item").Value;
readerContext.Add(itemStr);
// Remove min/max parameter info
var paramIndex = itemStr.IndexOf('(');
if (paramIndex > 0)
{
evaluatorContext.Add(String.Concat(itemStr.Substring(0, paramIndex + 1), ")"));
}
else
{
evaluatorContext.Add(itemStr);
}
}
ReaderContext = readerContext.ToArray();
EvaluatorContext = evaluatorContext.ToArray();
}
else if (String.Equals(definitionKey.Value, TemplateConstants.Description, StringComparison.Ordinal))
{
@@ -40,7 +58,17 @@ namespace GitHub.DistributedTask.ObjectTemplating.Schema
internal abstract DefinitionType DefinitionType { get; }
internal String[] Context { get; private set; } = new String[0];
/// <summary>
/// Used by the template reader to determine allowed expression values and functions.
/// Also used by the template reader to validate function min/max parameters.
/// </summary>
internal String[] ReaderContext { get; private set; } = new String[0];
/// <summary>
/// Used by the template evaluator to determine allowed expression values and functions.
/// The min/max parameter info is omitted.
/// </summary>
internal String[] EvaluatorContext { get; private set; } = new String[0];
internal abstract void Validate(
TemplateSchema schema,

2
src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/MappingDefinition.cs
View File

@@ -108,7 +108,7 @@ namespace GitHub.DistributedTask.ObjectTemplating.Schema
{
var inherited = schema.GetDefinition(Inherits);
if (inherited.Context.Length > 0)
if (inherited.ReaderContext.Length > 0)
{
throw new NotSupportedException($"Property '{TemplateConstants.Context}' is not supported on inhertied definitions");
}

2
src/Sdk/DTObjectTemplating/ObjectTemplating/Schema/OneOfDefinition.cs
View File

@@ -62,7 +62,7 @@ namespace GitHub.DistributedTask.ObjectTemplating.Schema
{
var nestedDefinition = schema.GetDefinition(nestedType);
if (nestedDefinition.Context.Length > 0)
if (nestedDefinition.ReaderContext.Length > 0)
{
throw new ArgumentException($"'{name}' is a one-of definition and references another definition that defines context. This is currently not supported.");
}

20
src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateEvaluator.cs
View File

@@ -47,7 +47,16 @@ namespace GitHub.DistributedTask.ObjectTemplating
var evaluator = new TemplateEvaluator(context, template, removeBytes);
try
{
var availableContext = new HashSet<String>(context.ExpressionValues.Keys.Concat(context.ExpressionFunctions.Select(x => $"{x.Name}({x.MinParameters},{x.MaxParameters})")));
var availableContext = new HashSet<String>(StringComparer.OrdinalIgnoreCase);
foreach (var key in context.ExpressionValues.Keys)
{
availableContext.Add(key);
}
foreach (var function in context.ExpressionFunctions)
{
availableContext.Add($"{function.Name}()");
}
var definitionInfo = new DefinitionInfo(context.Schema, type, availableContext);
result = evaluator.Evaluate(definitionInfo);
@@ -393,14 +402,13 @@ namespace GitHub.DistributedTask.ObjectTemplating
Definition = m_schema.GetDefinition(name);
// Determine whether to expand
if (Definition.Context.Length > 0)
m_allowedContext = Definition.EvaluatorContext;
if (Definition.EvaluatorContext.Length > 0)
{
m_allowedContext = Definition.Context;
Expand = m_availableContext.IsSupersetOf(m_allowedContext);
}
else
{
m_allowedContext = new String[0];
Expand = false;
}
}
@@ -416,9 +424,9 @@ namespace GitHub.DistributedTask.ObjectTemplating
Definition = m_schema.GetDefinition(name);
// Determine whether to expand
if (Definition.Context.Length > 0)
if (Definition.EvaluatorContext.Length > 0)
{
m_allowedContext = new HashSet<String>(parent.m_allowedContext.Concat(Definition.Context)).ToArray();
m_allowedContext = new HashSet<String>(parent.m_allowedContext.Concat(Definition.EvaluatorContext), StringComparer.OrdinalIgnoreCase).ToArray();
Expand = m_availableContext.IsSupersetOf(m_allowedContext);
}
else

8
src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateException.cs
View File

@@ -49,6 +49,14 @@ namespace GitHub.DistributedTask.ObjectTemplating
m_errors = new List<TemplateValidationError>(errors ?? Enumerable.Empty<TemplateValidationError>());
}
public TemplateValidationException(
String message,
IEnumerable<TemplateValidationError> errors)
: this(message)
{
m_errors = new List<TemplateValidationError>(errors ?? Enumerable.Empty<TemplateValidationError>());
}
public TemplateValidationException(String message)
: base(message)
{

17
src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateReader.cs
View File

@@ -780,15 +780,8 @@ namespace GitHub.DistributedTask.ObjectTemplating
// Lookup the definition
Definition = m_schema.GetDefinition(name);
// Determine whether to expand
if (Definition.Context.Length > 0)
{
AllowedContext = Definition.Context;
}
else
{
AllowedContext = new String[0];
}
// Record allowed context
AllowedContext = Definition.ReaderContext;
}
public DefinitionInfo(
@@ -800,10 +793,10 @@ namespace GitHub.DistributedTask.ObjectTemplating
// Lookup the definition
Definition = m_schema.GetDefinition(name);
// Determine whether to expand
if (Definition.Context.Length > 0)
// Record allowed context
if (Definition.ReaderContext.Length > 0)
{
AllowedContext = new HashSet<String>(parent.AllowedContext.Concat(Definition.Context)).ToArray();
AllowedContext = new HashSet<String>(parent.AllowedContext.Concat(Definition.ReaderContext), StringComparer.OrdinalIgnoreCase).ToArray();
}
else
{

20
src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateValidationErrors.cs
View File

@@ -2,6 +2,7 @@
using System.Collections;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Runtime.Serialization;
namespace GitHub.DistributedTask.ObjectTemplating
@@ -41,7 +42,7 @@ namespace GitHub.DistributedTask.ObjectTemplating
{
for (int i = 0; i < 50; i++)
{
String message = !String.IsNullOrEmpty(messagePrefix) ? $"{messagePrefix} {ex.Message}" : ex.Message;
String message = !String.IsNullOrEmpty(messagePrefix) ? $"{messagePrefix} {ex.Message}" : ex.ToString();
Add(new TemplateValidationError(message));
if (ex.InnerException == null)
{
@@ -88,6 +89,23 @@ namespace GitHub.DistributedTask.ObjectTemplating
}
}
/// <summary>
/// Throws <c ref="TemplateValidationException" /> if any errors.
/// <param name="prefix">The error message prefix</param>
/// </summary>
public void Check(String prefix)
{
if (String.IsNullOrEmpty(prefix))
{
this.Check();
}
else if (m_errors.Count > 0)
{
var message = $"{prefix.Trim()} {String.Join(",", m_errors.Select(e => e.Message))}";
throw new TemplateValidationException(message, m_errors);
}
}
public void Clear()
{
m_errors.Clear();

43
src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/ExpressionToken.cs
View File

@@ -1,7 +1,9 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Globalization;
using System.Runtime.Serialization;
using System.Text.RegularExpressions;
using GitHub.DistributedTask.Expressions2;
using GitHub.DistributedTask.Expressions2.Sdk;
using GitHub.Services.WebApi.Internal;
@@ -35,11 +37,29 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens
String[] allowedContext,
out Exception ex)
{
// Create dummy allowed contexts
INamedValueInfo[] namedValues = null;
// Create dummy named values and functions
var namedValues = new List<INamedValueInfo>();
var functions = new List<IFunctionInfo>();
if (allowedContext?.Length > 0)
{
namedValues = allowedContext.Select(x => new NamedValueInfo<ContextValueNode>(x)).ToArray();
foreach (var contextItem in allowedContext)
{
var match = s_function.Match(contextItem);
if (match.Success)
{
var functionName = match.Groups[1].Value;
var minParameters = Int32.Parse(match.Groups[2].Value, NumberStyles.None, CultureInfo.InvariantCulture);
var maxParametersRaw = match.Groups[3].Value;
var maxParameters = String.Equals(maxParametersRaw, TemplateConstants.MaxConstant, StringComparison.Ordinal)
? Int32.MaxValue
: Int32.Parse(maxParametersRaw, NumberStyles.None, CultureInfo.InvariantCulture);
functions.Add(new FunctionInfo<DummyFunction>(functionName, minParameters, maxParameters));
}
else
{
namedValues.Add(new NamedValueInfo<ContextValueNode>(contextItem));
}
}
}
// Parse
@@ -47,7 +67,7 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens
ExpressionNode root = null;
try
{
root = new ExpressionParser().CreateTree(expression, null, namedValues, null) as ExpressionNode;
root = new ExpressionParser().CreateTree(expression, null, namedValues, functions) as ExpressionNode;
result = true;
ex = null;
@@ -60,5 +80,18 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens
return result;
}
private sealed class DummyFunction : Function
{
protected override Object EvaluateCore(
EvaluationContext context,
out ResultMemory resultMemory)
{
resultMemory = null;
return null;
}
}
private static readonly Regex s_function = new Regex(@"^([a-zA-Z0-9_]+)\(([0-9]+),([0-9]+|MAX)\)$", RegexOptions.Compiled);
}
}

40
src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateTokenExtensions.cs
View File

@@ -1,5 +1,8 @@
using System;
using System.Collections.Generic;
using System.Linq;
using GitHub.DistributedTask.Expressions2;
using GitHub.DistributedTask.Expressions2.Sdk;
namespace GitHub.DistributedTask.ObjectTemplating.Tokens
{
@@ -106,6 +109,43 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens
throw new ArgumentException($"Error while reading '{objectDescription}'. Unexpected value '{literal.ToString()}'");
}
/// <summary>
/// Traverses the token and checks whether all required expression values
/// and functions are provided.
/// </summary>
public static bool CheckHasRequiredContext(
this TemplateToken token,
IReadOnlyObject expressionValues,
IList<IFunctionInfo> expressionFunctions)
{
var expressionTokens = token.Traverse()
.OfType<BasicExpressionToken>()
.ToArray();
var parser = new ExpressionParser();
foreach (var expressionToken in expressionTokens)
{
var tree = parser.ValidateSyntax(expressionToken.Expression, null);
foreach (var node in tree.Traverse())
{
if (node is NamedValue namedValue)
{
if (expressionValues?.Keys.Any(x => string.Equals(x, namedValue.Name, StringComparison.OrdinalIgnoreCase)) != true)
{
return false;
}
}
else if (node is Function function &&
!ExpressionConstants.WellKnownFunctions.ContainsKey(function.Name) &&
expressionFunctions?.Any(x => string.Equals(x.Name, function.Name, StringComparison.OrdinalIgnoreCase)) != true)
{
return false;
}
}
}
return true;
}
/// <summary>
/// Returns all tokens (depth first)
/// </summary>

3
src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConstants.cs
View File

@@ -24,6 +24,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public const String FetchDepth = "fetch-depth";
public const String GeneratedId = "generated-id";
public const String GitHub = "github";
public const String HashFiles = "hashFiles";
public const String Id = "id";
public const String If = "if";
public const String Image = "image";
@@ -31,6 +32,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public const String Inputs = "inputs";
public const String Job = "job";
public const String JobDefaultsRun = "job-defaults-run";
public const String JobIfResult = "job-if-result";
public const String JobOutputs = "job-outputs";
public const String Jobs = "jobs";
public const String Labels = "labels";
@@ -60,6 +62,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public const String Shell = "shell";
public const String Skipped = "skipped";
public const String StepEnv = "step-env";
public const String StepIfResult = "step-if-result";
public const String Steps = "steps";
public const String StepsScopeInputs = "steps-scope-inputs";
public const String StepsScopeOutputs = "steps-scope-outputs";

14
src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs
View File

@@ -16,6 +16,20 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
{
internal static class PipelineTemplateConverter
{
internal static Boolean ConvertToIfResult(
TemplateContext context,
TemplateToken ifResult)
{
var expression = ifResult.Traverse().FirstOrDefault(x => x is ExpressionToken);
if (expression != null)
{
throw new ArgumentException($"Unexpected type '{expression.GetType().Name}' encountered while reading 'if'.");
}
var evaluationResult = EvaluationResult.CreateIntermediateResult(null, ifResult);
return evaluationResult.IsTruthy;
}
internal static Boolean? ConvertToStepContinueOnError(
TemplateContext context,
TemplateToken token,

215
src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs
View File

@@ -4,7 +4,7 @@ using System.ComponentModel;
using System.Linq;
using System.Threading;
using GitHub.DistributedTask.Expressions2;
using GitHub.DistributedTask.Expressions2.Sdk;
using GitHub.DistributedTask.Expressions2.Sdk.Functions;
using GitHub.DistributedTask.ObjectTemplating;
using GitHub.DistributedTask.ObjectTemplating.Schema;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
@@ -14,6 +14,9 @@ using ITraceWriter = GitHub.DistributedTask.ObjectTemplating.ITraceWriter;
namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
{
/// <summary>
/// Evaluates parts of the workflow DOM. For example, a job strategy or step inputs.
/// </summary>
[EditorBrowsable(EditorBrowsableState.Never)]
public class PipelineTemplateEvaluator
{
@@ -50,13 +53,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public DictionaryContextData EvaluateStepScopeInputs(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(DictionaryContextData);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepsScopeInputs, token, 0, null, omitHeader: true);
@@ -76,13 +80,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public DictionaryContextData EvaluateStepScopeOutputs(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(DictionaryContextData);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepsScopeOutputs, token, 0, null, omitHeader: true);
@@ -102,13 +107,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public Boolean EvaluateStepContinueOnError(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(Boolean?);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.BooleanStepsContext, token, 0, null, omitHeader: true);
@@ -126,16 +132,44 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
return result ?? false;
}
public String EvaluateStepDisplayName(
TemplateToken token,
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(String);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StringStepsContext, token, 0, null, omitHeader: true);
context.Errors.Check();
result = PipelineTemplateConverter.ConvertToStepDisplayName(context, token);
}
catch (Exception ex) when (!(ex is TemplateValidationException))
{
context.Errors.Add(ex);
}
context.Errors.Check();
}
return result;
}
public Dictionary<String, String> EvaluateStepEnvironment(
TemplateToken token,
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions,
StringComparer keyComparer)
{
var result = default(Dictionary<String, String>);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepEnv, token, 0, null, omitHeader: true);
@@ -153,15 +187,44 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
return result ?? new Dictionary<String, String>(keyComparer);
}
public Boolean EvaluateStepIf(
TemplateToken token,
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions,
IEnumerable<KeyValuePair<String, Object>> expressionState)
{
var result = default(Boolean?);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData, expressionFunctions, expressionState);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepIfResult, token, 0, null, omitHeader: true);
context.Errors.Check();
result = PipelineTemplateConverter.ConvertToIfResult(context, token);
}
catch (Exception ex) when (!(ex is TemplateValidationException))
{
context.Errors.Add(ex);
}
context.Errors.Check();
}
return result ?? throw new InvalidOperationException("Step if cannot be null");
}
public Dictionary<String, String> EvaluateStepInputs(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(Dictionary<String, String>);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepWith, token, 0, null, omitHeader: true);
@@ -181,13 +244,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public Int32 EvaluateStepTimeout(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(Int32?);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.NumberStepsContext, token, 0, null, omitHeader: true);
@@ -207,13 +271,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public JobContainer EvaluateJobContainer(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(JobContainer);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.Container, token, 0, null, omitHeader: true);
@@ -233,13 +298,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public Dictionary<String, String> EvaluateJobOutput(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(Dictionary<String, String>);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.JobOutputs, token, 0, null, omitHeader: true);
@@ -269,13 +335,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public Dictionary<String, String> EvaluateJobDefaultsRun(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(Dictionary<String, String>);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.JobDefaultsRun, token, 0, null, omitHeader: true);
@@ -305,13 +372,14 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public IList<KeyValuePair<String, JobContainer>> EvaluateJobServiceContainers(
TemplateToken token,
DictionaryContextData contextData)
DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions)
{
var result = default(List<KeyValuePair<String, JobContainer>>);
if (token != null && token.Type != TokenType.Null)
{
var context = CreateContext(contextData);
var context = CreateContext(contextData, expressionFunctions);
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.Services, token, 0, null, omitHeader: true);
@@ -329,62 +397,10 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
return result;
}
public Boolean TryEvaluateStepDisplayName(
TemplateToken token,
private TemplateContext CreateContext(
DictionaryContextData contextData,
out String stepName)
{
stepName = default(String);
var context = CreateContext(contextData);
if (token != null && token.Type != TokenType.Null)
{
// We should only evaluate basic expressions if we are sure we have context on all the Named Values and functions
// Otherwise return and use a default name
if (token is BasicExpressionToken expressionToken)
{
ExpressionNode root = null;
try
{
root = new ExpressionParser().ValidateSyntax(expressionToken.Expression, null) as ExpressionNode;
}
catch (Exception exception)
{
context.Errors.Add(exception);
context.Errors.Check();
}
foreach (var node in root.Traverse())
{
if (node is NamedValue namedValue && !contextData.ContainsKey(namedValue.Name))
{
return false;
}
else if (node is Function function &&
!context.ExpressionFunctions.Any(item => String.Equals(item.Name, function.Name)) &&
!ExpressionConstants.WellKnownFunctions.ContainsKey(function.Name))
{
return false;
}
}
}
try
{
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StringStepsContext, token, 0, null, omitHeader: true);
context.Errors.Check();
stepName = PipelineTemplateConverter.ConvertToStepDisplayName(context, token);
}
catch (Exception ex) when (!(ex is TemplateValidationException))
{
context.Errors.Add(ex);
}
context.Errors.Check();
}
return true;
}
private TemplateContext CreateContext(DictionaryContextData contextData)
IList<IFunctionInfo> expressionFunctions,
IEnumerable<KeyValuePair<String, Object>> expressionState = null)
{
var result = new TemplateContext
{
@@ -407,7 +423,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
}
}
// Add named context
// Add named values
if (contextData != null)
{
foreach (var pair in contextData)
@@ -416,14 +432,46 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
}
}
// Compat for new agent against old server
foreach (var name in s_contextNames)
// Add functions
var functionNames = new HashSet<String>(StringComparer.OrdinalIgnoreCase);
if (expressionFunctions?.Count > 0)
{
foreach (var function in expressionFunctions)
{
result.ExpressionFunctions.Add(function);
functionNames.Add(function.Name);
}
}
// Add missing expression values and expression functions.
// This solves the following problems:
// - Compat for new agent against old server (new contexts not sent down in job message)
// - Evaluating early when all referenced contexts are available, even though all allowed
// contexts may not yet be available. For example, evaluating step display name can often
// be performed early.
foreach (var name in s_expressionValueNames)
{
if (!result.ExpressionValues.ContainsKey(name))
{
result.ExpressionValues[name] = null;
}
}
foreach (var name in s_expressionFunctionNames)
{
if (!functionNames.Contains(name))
{
result.ExpressionFunctions.Add(new FunctionInfo<NoOperation>(name, 0, Int32.MaxValue));
}
}
// Add state
if (expressionState != null)
{
foreach (var pair in expressionState)
{
result.State[pair.Key] = pair.Value;
}
}
return result;
}
@@ -431,9 +479,10 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
private readonly ITraceWriter m_trace;
private readonly TemplateSchema m_schema;
private readonly IList<String> m_fileTable;
private readonly String[] s_contextNames = new[]
private readonly String[] s_expressionValueNames = new[]
{
PipelineTemplateConstants.GitHub,
PipelineTemplateConstants.Needs,
PipelineTemplateConstants.Strategy,
PipelineTemplateConstants.Matrix,
PipelineTemplateConstants.Needs,
@@ -444,5 +493,13 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
PipelineTemplateConstants.Runner,
PipelineTemplateConstants.Env,
};
private readonly String[] s_expressionFunctionNames = new[]
{
PipelineTemplateConstants.Always,
PipelineTemplateConstants.Cancelled,
PipelineTemplateConstants.Failure,
PipelineTemplateConstants.HashFiles,
PipelineTemplateConstants.Success,
};
}
}

28
src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateSchemaFactory.cs
View File

@@ -2,25 +2,35 @@
using System.ComponentModel;
using System.IO;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using GitHub.DistributedTask.ObjectTemplating.Schema;
namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
{
[EditorBrowsable(EditorBrowsableState.Never)]
public sealed class PipelineTemplateSchemaFactory
public static class PipelineTemplateSchemaFactory
{
public TemplateSchema CreateSchema()
public static TemplateSchema GetSchema()
{
var assembly = Assembly.GetExecutingAssembly();
var json = default(String);
using (var stream = assembly.GetManifestResourceStream("GitHub.DistributedTask.Pipelines.ObjectTemplating.workflow-v1.0.json"))
using (var streamReader = new StreamReader(stream))
if (s_schema == null)
{
json = streamReader.ReadToEnd();
var assembly = Assembly.GetExecutingAssembly();
var json = default(String);
using (var stream = assembly.GetManifestResourceStream("GitHub.DistributedTask.Pipelines.ObjectTemplating.workflow-v1.0.json"))
using (var streamReader = new StreamReader(stream))
{
json = streamReader.ReadToEnd();
}
var objectReader = new JsonObjectReader(null, json);
var schema = TemplateSchema.Load(objectReader);
Interlocked.CompareExchange(ref s_schema, schema, null);
}
var objectReader = new JsonObjectReader(null, json);
return TemplateSchema.Load(objectReader);
return s_schema;
}
private static TemplateSchema s_schema;
}
}

212
src/Sdk/DTPipelines/workflow-v1.0.json
View File

@@ -38,8 +38,8 @@
"steps-scope-input-value": {
"context": [
"github",
"strategy",
"needs",
"strategy",
"matrix",
"secrets",
"steps",
@@ -66,9 +66,9 @@
"steps-scope-output-value": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
@@ -91,9 +91,9 @@
"description": "Default input values for a steps template",
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"one-of": [
"string",
@@ -114,9 +114,9 @@
"description": "Output values for a steps template",
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
@@ -204,6 +204,25 @@
"string": {}
},
"job-if-result": {
"context": [
"github",
"needs",
"always(0,0)",
"failure(0,MAX)",
"cancelled(0,0)",
"success(0,MAX)"
],
"one-of": [
"null",
"boolean",
"number",
"string",
"sequence",
"mapping"
]
},
"strategy": {
"context": [
"github",
@@ -272,9 +291,9 @@
"runs-on": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"one-of": [
"non-empty-string",
@@ -297,10 +316,10 @@
"job-env": {
"context": [
"github",
"secrets",
"needs",
"strategy",
"matrix",
"needs"
"secrets"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -444,9 +463,9 @@
"step-if": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"steps",
"job",
"runner",
@@ -454,7 +473,8 @@
"always(0,0)",
"failure(0,0)",
"cancelled(0,0)",
"success(0,0)"
"success(0,0)",
"hashFiles(1,255)"
],
"string": {}
},
@@ -462,9 +482,9 @@
"step-if-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"steps",
"inputs",
"job",
@@ -473,11 +493,63 @@
"always(0,0)",
"failure(0,0)",
"cancelled(0,0)",
"success(0,0)"
"success(0,0)",
"hashFiles(1,255)"
],
"string": {}
},
"step-if-result": {
"context": [
"github",
"strategy",
"matrix",
"steps",
"job",
"runner",
"env",
"always(0,0)",
"failure(0,0)",
"cancelled(0,0)",
"success(0,0)",
"hashFiles(1,255)"
],
"one-of": [
"null",
"boolean",
"number",
"string",
"sequence",
"mapping"
]
},
"step-if-result-in-template": {
"context": [
"github",
"strategy",
"matrix",
"steps",
"inputs",
"job",
"runner",
"env",
"always(0,0)",
"failure(0,0)",
"cancelled(0,0)",
"success(0,0)",
"hashFiles(1,255)"
],
"one-of": [
"null",
"boolean",
"number",
"string",
"sequence",
"mapping"
]
},
"steps-template-reference": {
"mapping": {
"properties": {
@@ -501,9 +573,9 @@
"steps-template-reference-inputs": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
@@ -519,9 +591,9 @@
"steps-template-reference-inputs-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
@@ -538,14 +610,15 @@
"step-env": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -556,15 +629,16 @@
"step-env-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -575,14 +649,35 @@
"step-with": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"mapping": {
"loose-key-type": "non-empty-string",
"loose-value-type": "string"
}
},
"step-with-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"secrets",
"steps",
"inputs",
"job",
"runner",
"env",
"hashFiles(1,255)"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -593,9 +688,9 @@
"container": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"one-of": [
"string",
@@ -618,9 +713,9 @@
"services": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"mapping": {
"loose-key-type": "non-empty-string",
@@ -631,9 +726,9 @@
"services-container": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"one-of": [
"non-empty-string",
@@ -648,25 +743,6 @@
}
},
"step-with-in-template": {
"context": [
"github",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
"job",
"runner",
"env"
],
"mapping": {
"loose-key-type": "non-empty-string",
"loose-value-type": "string"
}
},
"non-empty-string": {
"string": {
"require-non-empty": true
@@ -682,9 +758,9 @@
"boolean-strategy-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"boolean": {}
},
@@ -692,9 +768,9 @@
"number-strategy-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"number": {}
},
@@ -702,9 +778,9 @@
"string-strategy-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs"
"matrix"
],
"string": {}
},
@@ -712,14 +788,15 @@
"boolean-steps-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"boolean": {}
},
@@ -727,15 +804,16 @@
"boolean-steps-context-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"boolean": {}
},
@@ -743,14 +821,15 @@
"number-steps-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"number": {}
},
@@ -758,15 +837,16 @@
"number-steps-context-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"number": {}
},
@@ -774,9 +854,9 @@
"string-runner-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
@@ -789,14 +869,15 @@
"string-steps-context": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"string": {}
},
@@ -804,15 +885,16 @@
"string-steps-context-in-template": {
"context": [
"github",
"needs",
"strategy",
"matrix",
"needs",
"secrets",
"steps",
"inputs",
"job",
"runner",
"env"
"env",
"hashFiles(1,255)"
],
"string": {}
}