From 0cba42590f47d73d811a8bb25823e78a42bd766f Mon Sep 17 00:00:00 2001 From: eric sciple Date: Tue, 3 Mar 2020 22:38:19 -0500 Subject: [PATCH] preserve workflow file/line/column for better error messages (#356) --- src/Runner.Worker/ActionManifestManager.cs | 574 +----------------- src/Runner.Worker/ActionRunner.cs | 7 +- src/Runner.Worker/ExecutionContext.cs | 14 + src/Runner.Worker/JobExtension.cs | 4 +- src/Runner.Worker/StepsRunner.cs | 17 +- .../ObjectTemplating/TemplateContext.cs | 12 +- .../ObjectTemplating/Tokens/TemplateToken.cs | 8 +- .../Tokens/TemplateTokenJsonConverter.cs | 65 +- .../Pipelines/AgentJobRequestMessage.cs | 28 +- .../PipelineTemplateEvaluator.cs | 15 +- .../ObjectTemplating/YamlObjectReader.cs | 572 +++++++++++++++++ src/Test/L0/Listener/JobDispatcherL0.cs | 2 +- src/Test/L0/Listener/RunnerL0.cs | 2 +- src/Test/L0/Worker/ActionCommandManagerL0.cs | 2 +- src/Test/L0/Worker/ExecutionContextL0.cs | 6 +- src/Test/L0/Worker/JobExtensionL0.cs | 2 +- src/Test/L0/Worker/JobRunnerL0.cs | 2 +- src/Test/L0/Worker/WorkerL0.cs | 2 +- 18 files changed, 721 insertions(+), 613 deletions(-) create mode 100644 src/Sdk/DTPipelines/Pipelines/ObjectTemplating/YamlObjectReader.cs diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 82e4c2ae9..980b87e17 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -32,6 +32,7 @@ namespace GitHub.Runner.Worker public sealed class ActionManifestManager : RunnerService, IActionManifestManager { private TemplateSchema _actionManifestSchema; + private IReadOnlyList _fileTable; public override void Initialize(IHostContext hostContext) { @@ -61,6 +62,9 @@ namespace GitHub.Runner.Worker // Get the file ID var fileId = context.GetFileId(manifestFile); + _fileTable = context.GetFileTable(); + + // Read the file var fileContent = File.ReadAllText(manifestFile); using (var stringReader = new StringReader(fileContent)) { @@ -265,6 +269,15 @@ namespace GitHub.Runner.Worker } } + // Add the file table + if (_fileTable?.Count > 0) + { + for (var i = 0 ; i < _fileTable.Count ; i++) + { + result.GetFileId(_fileTable[i]); + } + } + return result; } @@ -415,566 +428,5 @@ namespace GitHub.Runner.Worker } } } - - /// - /// Converts a YAML file into a TemplateToken - /// - internal sealed class YamlObjectReader : IObjectReader - { - internal YamlObjectReader( - Int32? fileId, - TextReader input) - { - m_fileId = fileId; - m_parser = new Parser(input); - } - - public Boolean AllowLiteral(out LiteralToken value) - { - if (EvaluateCurrent() is Scalar scalar) - { - // Tag specified - if (!string.IsNullOrEmpty(scalar.Tag)) - { - // String tag - if (string.Equals(scalar.Tag, c_stringTag, StringComparison.Ordinal)) - { - value = new StringToken(m_fileId, scalar.Start.Line, scalar.Start.Column, scalar.Value); - MoveNext(); - return true; - } - - // Not plain style - if (scalar.Style != ScalarStyle.Plain) - { - throw new NotSupportedException($"The scalar style '{scalar.Style}' on line {scalar.Start.Line} and column {scalar.Start.Column} is not valid with the tag '{scalar.Tag}'"); - } - - // Boolean, Float, Integer, or Null - switch (scalar.Tag) - { - case c_booleanTag: - value = ParseBoolean(scalar); - break; - case c_floatTag: - value = ParseFloat(scalar); - break; - case c_integerTag: - value = ParseInteger(scalar); - break; - case c_nullTag: - value = ParseNull(scalar); - break; - default: - throw new NotSupportedException($"Unexpected tag '{scalar.Tag}'"); - } - - MoveNext(); - return true; - } - - // Plain style, determine type using YAML 1.2 "core" schema https://yaml.org/spec/1.2/spec.html#id2804923 - if (scalar.Style == ScalarStyle.Plain) - { - if (MatchNull(scalar, out var nullToken)) - { - value = nullToken; - } - else if (MatchBoolean(scalar, out var booleanToken)) - { - value = booleanToken; - } - else if (MatchInteger(scalar, out var numberToken) || - MatchFloat(scalar, out numberToken)) - { - value = numberToken; - } - else - { - value = new StringToken(m_fileId, scalar.Start.Line, scalar.Start.Column, scalar.Value); - } - - MoveNext(); - return true; - } - - // Otherwise assume string - value = new StringToken(m_fileId, scalar.Start.Line, scalar.Start.Column, scalar.Value); - MoveNext(); - return true; - } - - value = default; - return false; - } - - public Boolean AllowSequenceStart(out SequenceToken value) - { - if (EvaluateCurrent() is SequenceStart sequenceStart) - { - value = new SequenceToken(m_fileId, sequenceStart.Start.Line, sequenceStart.Start.Column); - MoveNext(); - return true; - } - - value = default; - return false; - } - - public Boolean AllowSequenceEnd() - { - if (EvaluateCurrent() is SequenceEnd) - { - MoveNext(); - return true; - } - - return false; - } - - public Boolean AllowMappingStart(out MappingToken value) - { - if (EvaluateCurrent() is MappingStart mappingStart) - { - value = new MappingToken(m_fileId, mappingStart.Start.Line, mappingStart.Start.Column); - MoveNext(); - return true; - } - - value = default; - return false; - } - - public Boolean AllowMappingEnd() - { - if (EvaluateCurrent() is MappingEnd) - { - MoveNext(); - return true; - } - - return false; - } - - /// - /// Consumes the last parsing events, which are expected to be DocumentEnd and StreamEnd. - /// - public void ValidateEnd() - { - if (EvaluateCurrent() is DocumentEnd) - { - MoveNext(); - } - else - { - throw new InvalidOperationException("Expected document end parse event"); - } - - if (EvaluateCurrent() is StreamEnd) - { - MoveNext(); - } - else - { - throw new InvalidOperationException("Expected stream end parse event"); - } - - if (MoveNext()) - { - throw new InvalidOperationException("Expected end of parse events"); - } - } - - /// - /// Consumes the first parsing events, which are expected to be StreamStart and DocumentStart. - /// - public void ValidateStart() - { - if (EvaluateCurrent() != null) - { - throw new InvalidOperationException("Unexpected parser state"); - } - - if (!MoveNext()) - { - throw new InvalidOperationException("Expected a parse event"); - } - - if (EvaluateCurrent() is StreamStart) - { - MoveNext(); - } - else - { - throw new InvalidOperationException("Expected stream start parse event"); - } - - if (EvaluateCurrent() is DocumentStart) - { - MoveNext(); - } - else - { - throw new InvalidOperationException("Expected document start parse event"); - } - } - - private ParsingEvent EvaluateCurrent() - { - if (m_current == null) - { - m_current = m_parser.Current; - if (m_current != null) - { - if (m_current is Scalar scalar) - { - // Verify not using achors - if (scalar.Anchor != null) - { - throw new InvalidOperationException($"Anchors are not currently supported. Remove the anchor '{scalar.Anchor}'"); - } - } - else if (m_current is MappingStart mappingStart) - { - // Verify not using achors - if (mappingStart.Anchor != null) - { - throw new InvalidOperationException($"Anchors are not currently supported. Remove the anchor '{mappingStart.Anchor}'"); - } - } - else if (m_current is SequenceStart sequenceStart) - { - // Verify not using achors - if (sequenceStart.Anchor != null) - { - throw new InvalidOperationException($"Anchors are not currently supported. Remove the anchor '{sequenceStart.Anchor}'"); - } - } - else if (!(m_current is MappingEnd) && - !(m_current is SequenceEnd) && - !(m_current is DocumentStart) && - !(m_current is DocumentEnd) && - !(m_current is StreamStart) && - !(m_current is StreamEnd)) - { - throw new InvalidOperationException($"Unexpected parsing event type: {m_current.GetType().Name}"); - } - } - } - - return m_current; - } - - private Boolean MoveNext() - { - m_current = null; - return m_parser.MoveNext(); - } - - private BooleanToken ParseBoolean(Scalar scalar) - { - if (MatchBoolean(scalar, out var token)) - { - return token; - } - - ThrowInvalidValue(scalar, c_booleanTag); // throws - return default; - } - - private NumberToken ParseFloat(Scalar scalar) - { - if (MatchFloat(scalar, out var token)) - { - return token; - } - - ThrowInvalidValue(scalar, c_floatTag); // throws - return default; - } - - private NumberToken ParseInteger(Scalar scalar) - { - if (MatchInteger(scalar, out var token)) - { - return token; - } - - ThrowInvalidValue(scalar, c_integerTag); // throws - return default; - } - - private NullToken ParseNull(Scalar scalar) - { - if (MatchNull(scalar, out var token)) - { - return token; - } - - ThrowInvalidValue(scalar, c_nullTag); // throws - return default; - } - - private Boolean MatchBoolean( - Scalar scalar, - out BooleanToken value) - { - // YAML 1.2 "core" schema https://yaml.org/spec/1.2/spec.html#id2804923 - switch (scalar.Value ?? string.Empty) - { - case "true": - case "True": - case "TRUE": - value = new BooleanToken(m_fileId, scalar.Start.Line, scalar.Start.Column, true); - return true; - case "false": - case "False": - case "FALSE": - value = new BooleanToken(m_fileId, scalar.Start.Line, scalar.Start.Column, false); - return true; - } - - value = default; - return false; - } - - private Boolean MatchFloat( - Scalar scalar, - out NumberToken value) - { - // YAML 1.2 "core" schema https://yaml.org/spec/1.2/spec.html#id2804923 - var str = scalar.Value; - if (!string.IsNullOrEmpty(str)) - { - // Check for [-+]?(\.inf|\.Inf|\.INF)|\.nan|\.NaN|\.NAN - switch (str) - { - case ".inf": - case ".Inf": - case ".INF": - case "+.inf": - case "+.Inf": - case "+.INF": - value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, Double.PositiveInfinity); - return true; - case "-.inf": - case "-.Inf": - case "-.INF": - value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, Double.NegativeInfinity); - return true; - case ".nan": - case ".NaN": - case ".NAN": - value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, Double.NaN); - return true; - } - - // Otherwise check [-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)? - - // Skip leading sign - var index = str[0] == '-' || str[0] == '+' ? 1 : 0; - - // Check for integer portion - var length = str.Length; - var hasInteger = false; - while (index < length && str[index] >= '0' && str[index] <= '9') - { - hasInteger = true; - index++; - } - - // Check for decimal point - var hasDot = false; - if (index < length && str[index] == '.') - { - hasDot = true; - index++; - } - - // Check for decimal portion - var hasDecimal = false; - while (index < length && str[index] >= '0' && str[index] <= '9') - { - hasDecimal = true; - index++; - } - - // Check [-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?) - if ((hasDot && hasDecimal) || hasInteger) - { - // Check for end - if (index == length) - { - // Try parse - if (Double.TryParse(str, NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out var doubleValue)) - { - value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, doubleValue); - return true; - } - // Otherwise exceeds range - else - { - ThrowInvalidValue(scalar, c_floatTag); // throws - } - } - // Check [eE][-+]?[0-9] - else if (index < length && (str[index] == 'e' || str[index] == 'E')) - { - index++; - - // Skip sign - if (index < length && (str[index] == '-' || str[index] == '+')) - { - index++; - } - - // Check for exponent - var hasExponent = false; - while (index < length && str[index] >= '0' && str[index] <= '9') - { - hasExponent = true; - index++; - } - - // Check for end - if (hasExponent && index == length) - { - // Try parse - if (Double.TryParse(str, NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent, CultureInfo.InvariantCulture, out var doubleValue)) - { - value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, (Double)doubleValue); - return true; - } - // Otherwise exceeds range - else - { - ThrowInvalidValue(scalar, c_floatTag); // throws - } - } - } - } - } - - value = default; - return false; - } - - private Boolean MatchInteger( - Scalar scalar, - out NumberToken value) - { - // YAML 1.2 "core" schema https://yaml.org/spec/1.2/spec.html#id2804923 - var str = scalar.Value; - if (!string.IsNullOrEmpty(str)) - { - // Check for [0-9]+ - var firstChar = str[0]; - if (firstChar >= '0' && firstChar <= '9' && - str.Skip(1).All(x => x >= '0' && x <= '9')) - { - // Try parse - if (Double.TryParse(str, NumberStyles.None, CultureInfo.InvariantCulture, out var doubleValue)) - { - value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, doubleValue); - return true; - } - - // Otherwise exceeds range - ThrowInvalidValue(scalar, c_integerTag); // throws - } - // Check for (-|+)[0-9]+ - else if ((firstChar == '-' || firstChar == '+') && - str.Length > 1 && - str.Skip(1).All(x => x >= '0' && x <= '9')) - { - // Try parse - if (Double.TryParse(str, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var doubleValue)) - { - value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, doubleValue); - return true; - } - - // Otherwise exceeds range - ThrowInvalidValue(scalar, c_integerTag); // throws - } - // Check for 0x[0-9a-fA-F]+ - else if (firstChar == '0' && - str.Length > 2 && - str[1] == 'x' && - str.Skip(2).All(x => (x >= '0' && x <= '9') || (x >= 'a' && x <= 'f') || (x >= 'A' && x <= 'F'))) - { - // Try parse - if (Int32.TryParse(str.Substring(2), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out var integerValue)) - { - value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, integerValue); - return true; - } - - // Otherwise exceeds range - ThrowInvalidValue(scalar, c_integerTag); // throws - } - // Check for 0o[0-9]+ - else if (firstChar == '0' && - str.Length > 2 && - str[1] == 'o' && - str.Skip(2).All(x => x >= '0' && x <= '7')) - { - // Try parse - var integerValue = default(Int32); - try - { - integerValue = Convert.ToInt32(str.Substring(2), 8); - } - // Otherwise exceeds range - catch (Exception) - { - ThrowInvalidValue(scalar, c_integerTag); // throws - } - - value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, integerValue); - return true; - } - } - - value = default; - return false; - } - - private Boolean MatchNull( - Scalar scalar, - out NullToken value) - { - // YAML 1.2 "core" schema https://yaml.org/spec/1.2/spec.html#id2804923 - switch (scalar.Value ?? string.Empty) - { - case "": - case "null": - case "Null": - case "NULL": - case "~": - value = new NullToken(m_fileId, scalar.Start.Line, scalar.Start.Column); - return true; - } - - value = default; - return false; - } - - private void ThrowInvalidValue( - Scalar scalar, - String tag) - { - throw new NotSupportedException($"The value '{scalar.Value}' on line {scalar.Start.Line} and column {scalar.Start.Column} is invalid for the type '{scalar.Tag}'"); - } - - private const String c_booleanTag = "tag:yaml.org,2002:bool"; - private const String c_floatTag = "tag:yaml.org,2002:float"; - private const String c_integerTag = "tag:yaml.org,2002:int"; - private const String c_nullTag = "tag:yaml.org,2002:null"; - private const String c_stringTag = "tag:yaml.org,2002:string"; - private readonly Int32? m_fileId; - private readonly Parser m_parser; - private ParsingEvent m_current; - } } diff --git a/src/Runner.Worker/ActionRunner.cs b/src/Runner.Worker/ActionRunner.cs index c604679bc..3b7e83e95 100644 --- a/src/Runner.Worker/ActionRunner.cs +++ b/src/Runner.Worker/ActionRunner.cs @@ -141,9 +141,7 @@ namespace GitHub.Runner.Worker // Load the inputs. ExecutionContext.Debug("Loading inputs"); - var templateTrace = ExecutionContext.ToTemplateTraceWriter(); - var schema = new PipelineTemplateSchemaFactory().CreateSchema(); - var templateEvaluator = new PipelineTemplateEvaluator(templateTrace, schema); + var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator(); var inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, ExecutionContext.ExpressionValues); foreach (KeyValuePair input in inputs) @@ -295,8 +293,7 @@ namespace GitHub.Runner.Worker return displayName; } // Try evaluating fully - var schema = new PipelineTemplateSchemaFactory().CreateSchema(); - var templateEvaluator = new PipelineTemplateEvaluator(context.ToTemplateTraceWriter(), schema); + var templateEvaluator = context.ToPipelineTemplateEvaluator(); try { didFullyEvaluate = templateEvaluator.TryEvaluateStepDisplayName(tokenToParse, contextData, out displayName); diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 5c7bbfa7e..f68b98f51 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -11,6 +11,7 @@ using GitHub.Runner.Worker.Container; using GitHub.Services.WebApi; using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.Pipelines.ContextData; +using GitHub.DistributedTask.Pipelines.ObjectTemplating; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common.Util; using GitHub.Runner.Common; @@ -49,6 +50,7 @@ namespace GitHub.Runner.Worker HashSet OutputVariables { get; } IDictionary EnvironmentVariables { get; } IDictionary Scopes { get; } + IList FileTable { get; } StepsContext StepsContext { get; } DictionaryContextData ExpressionValues { get; } List PrependPath { get; } @@ -141,6 +143,7 @@ namespace GitHub.Runner.Worker public HashSet OutputVariables => _outputvariables; public IDictionary EnvironmentVariables { get; private set; } public IDictionary Scopes { get; private set; } + public IList FileTable { get; private set; } public StepsContext StepsContext { get; private set; } public DictionaryContextData ExpressionValues { get; } = new DictionaryContextData(); public bool WriteDebug { get; private set; } @@ -266,6 +269,7 @@ namespace GitHub.Runner.Worker } child.EnvironmentVariables = EnvironmentVariables; child.Scopes = Scopes; + child.FileTable = FileTable; child.StepsContext = StepsContext; foreach (var pair in ExpressionValues) { @@ -569,6 +573,9 @@ namespace GitHub.Runner.Worker } } + // File table + FileTable = new List(message.FileTable ?? new string[0]); + // Expression functions if (Variables.GetBoolean("System.HashFilesV2") == true) { @@ -886,6 +893,13 @@ namespace GitHub.Runner.Worker } } + public static PipelineTemplateEvaluator ToPipelineTemplateEvaluator(this IExecutionContext context) + { + var templateTrace = context.ToTemplateTraceWriter(); + var schema = new PipelineTemplateSchemaFactory().CreateSchema(); + return new PipelineTemplateEvaluator(templateTrace, schema, context.FileTable); + } + public static ObjectTemplating.ITraceWriter ToTemplateTraceWriter(this IExecutionContext context) { return new TemplateTraceWriter(context); diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 11834b8f3..d7b6a99b1 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -78,9 +78,7 @@ namespace GitHub.Runner.Worker // Evaluate the job-level environment variables context.Debug("Evaluating job-level environment variables"); - var templateTrace = context.ToTemplateTraceWriter(); - var schema = new PipelineTemplateSchemaFactory().CreateSchema(); - var templateEvaluator = new PipelineTemplateEvaluator(templateTrace, schema); + var templateEvaluator = context.ToPipelineTemplateEvaluator(); foreach (var token in message.EnvironmentVariables) { var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, VarUtil.EnvironmentVariableKeyComparer); diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index 82cb8b9aa..f69530672 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -98,9 +98,7 @@ namespace GitHub.Runner.Worker step.ExecutionContext.SetGitHubContext("action", actionStep.Action.Name); // Evaluate and merge action's env block to env context - var templateTrace = step.ExecutionContext.ToTemplateTraceWriter(); - var schema = new PipelineTemplateSchemaFactory().CreateSchema(); - var templateEvaluator = new PipelineTemplateEvaluator(templateTrace, schema); + var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(); var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, VarUtil.EnvironmentVariableKeyComparer); foreach (var env in actionEnvironment) { @@ -247,7 +245,7 @@ namespace GitHub.Runner.Worker // Set the timeout var timeoutMinutes = 0; - var templateEvaluator = CreateTemplateEvaluator(step.ExecutionContext); + var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(); try { timeoutMinutes = templateEvaluator.EvaluateStepTimeout(step.Timeout, step.ExecutionContext.ExpressionValues); @@ -389,7 +387,7 @@ namespace GitHub.Runner.Worker executionContext.Debug($"Initializing scope '{scope.Name}'"); executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.ParentName); executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null; - var templateEvaluator = CreateTemplateEvaluator(executionContext); + var templateEvaluator = executionContext.ToPipelineTemplateEvaluator(); var inputs = default(DictionaryContextData); try { @@ -445,7 +443,7 @@ namespace GitHub.Runner.Worker executionContext.Debug($"Finalizing scope '{scope.Name}'"); executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.Name); executionContext.ExpressionValues["inputs"] = null; - var templateEvaluator = CreateTemplateEvaluator(executionContext); + var templateEvaluator = executionContext.ToPipelineTemplateEvaluator(); var outputs = default(DictionaryContextData); try { @@ -477,12 +475,5 @@ namespace GitHub.Runner.Worker executionContext.Complete(result, resultCode: resultCode); } - - private PipelineTemplateEvaluator CreateTemplateEvaluator(IExecutionContext executionContext) - { - var templateTrace = executionContext.ToTemplateTraceWriter(); - var schema = new PipelineTemplateSchemaFactory().CreateSchema(); - return new PipelineTemplateEvaluator(templateTrace, schema); - } } } diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateContext.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateContext.cs index 34ea45071..af8cf76ec 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateContext.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/TemplateContext.cs @@ -184,6 +184,7 @@ namespace GitHub.DistributedTask.ObjectTemplating id = FileIds.Count + 1; FileIds.Add(file, id); FileNames.Add(file); + Memory.AddBytes(file); } return id; @@ -191,7 +192,12 @@ namespace GitHub.DistributedTask.ObjectTemplating internal String GetFileName(Int32 fileId) { - return FileNames[fileId - 1]; + return FileNames.Count >= fileId ? FileNames[fileId - 1] : null; + } + + internal IReadOnlyList GetFileTable() + { + return FileNames.AsReadOnly(); } private String GetErrorPrefix( @@ -199,9 +205,9 @@ namespace GitHub.DistributedTask.ObjectTemplating Int32? line, Int32? column) { - if (fileId != null) + var fileName = fileId.HasValue ? GetFileName(fileId.Value) : null; + if (!String.IsNullOrEmpty(fileName)) { - var fileName = GetFileName(fileId.Value); if (line != null && column != null) { return $"{fileName} {TemplateStrings.LineColumn(line, column)}:"; diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateToken.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateToken.cs index afe46356f..21a8aab78 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateToken.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateToken.cs @@ -30,14 +30,14 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens Column = column; } - [IgnoreDataMember] - internal Int32? FileId { get; set; } + [DataMember(Name = "file", EmitDefaultValue = false)] + internal Int32? FileId { get; private set; } [DataMember(Name = "line", EmitDefaultValue = false)] - internal Int32? Line { get; } + internal Int32? Line { get; private set; } [DataMember(Name = "col", EmitDefaultValue = false)] - internal Int32? Column { get; } + internal Int32? Column { get; private set; } [DataMember(Name = "type", EmitDefaultValue = false)] internal Int32 Type { get; } diff --git a/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateTokenJsonConverter.cs b/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateTokenJsonConverter.cs index 1520942be..eb6c96384 100644 --- a/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateTokenJsonConverter.cs +++ b/src/Sdk/DTObjectTemplating/ObjectTemplating/Tokens/TemplateTokenJsonConverter.cs @@ -115,13 +115,12 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens Object value, JsonSerializer serializer) { - base.WriteJson(writer, value, serializer); if (value is TemplateToken token) { switch (token.Type) { case TokenType.Null: - if (token.Line == null && token.Column == null) + if (token.FileId == null && token.Line == null && token.Column == null) { writer.WriteNull(); } @@ -130,12 +129,17 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens writer.WriteStartObject(); writer.WritePropertyName("type"); writer.WriteValue(token.Type); + if (token.FileId != null) + { + writer.WritePropertyName("file"); + writer.WriteValue(token.FileId); + } if (token.Line != null) { writer.WritePropertyName("line"); writer.WriteValue(token.Line); } - if (token.Line != null) + if (token.Column != null) { writer.WritePropertyName("col"); writer.WriteValue(token.Column); @@ -146,7 +150,7 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens case TokenType.Boolean: var booleanToken = token as BooleanToken; - if (token.Line == null && token.Column == null) + if (token.FileId == null && token.Line == null && token.Column == null) { writer.WriteValue(booleanToken.Value); } @@ -155,12 +159,17 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens writer.WriteStartObject(); writer.WritePropertyName("type"); writer.WriteValue(token.Type); + if (token.FileId != null) + { + writer.WritePropertyName("file"); + writer.WriteValue(token.FileId); + } if (token.Line != null) { writer.WritePropertyName("line"); writer.WriteValue(token.Line); } - if (token.Line != null) + if (token.Column != null) { writer.WritePropertyName("col"); writer.WriteValue(token.Column); @@ -173,7 +182,7 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens case TokenType.Number: var numberToken = token as NumberToken; - if (token.Line == null && token.Column == null) + if (token.FileId == null && token.Line == null && token.Column == null) { writer.WriteValue(numberToken.Value); } @@ -182,12 +191,17 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens writer.WriteStartObject(); writer.WritePropertyName("type"); writer.WriteValue(token.Type); + if (token.FileId != null) + { + writer.WritePropertyName("file"); + writer.WriteValue(token.FileId); + } if (token.Line != null) { writer.WritePropertyName("line"); writer.WriteValue(token.Line); } - if (token.Line != null) + if (token.Column != null) { writer.WritePropertyName("col"); writer.WriteValue(token.Column); @@ -200,7 +214,7 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens case TokenType.String: var stringToken = token as StringToken; - if (token.Line == null && token.Column == null) + if (token.FileId == null && token.Line == null && token.Column == null) { writer.WriteValue(stringToken.Value); } @@ -209,12 +223,17 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens writer.WriteStartObject(); writer.WritePropertyName("type"); writer.WriteValue(token.Type); + if (token.FileId != null) + { + writer.WritePropertyName("file"); + writer.WriteValue(token.FileId); + } if (token.Line != null) { writer.WritePropertyName("line"); writer.WriteValue(token.Line); } - if (token.Line != null) + if (token.Column != null) { writer.WritePropertyName("col"); writer.WriteValue(token.Column); @@ -230,12 +249,17 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens writer.WriteStartObject(); writer.WritePropertyName("type"); writer.WriteValue(token.Type); + if (token.FileId != null) + { + writer.WritePropertyName("file"); + writer.WriteValue(token.FileId); + } if (token.Line != null) { writer.WritePropertyName("line"); writer.WriteValue(token.Line); } - if (token.Line != null) + if (token.Column != null) { writer.WritePropertyName("col"); writer.WriteValue(token.Column); @@ -253,12 +277,17 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens writer.WriteStartObject(); writer.WritePropertyName("type"); writer.WriteValue(token.Type); + if (token.FileId != null) + { + writer.WritePropertyName("file"); + writer.WriteValue(token.FileId); + } if (token.Line != null) { writer.WritePropertyName("line"); writer.WriteValue(token.Line); } - if (token.Line != null) + if (token.Column != null) { writer.WritePropertyName("col"); writer.WriteValue(token.Column); @@ -273,12 +302,17 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens writer.WriteStartObject(); writer.WritePropertyName("type"); writer.WriteValue(token.Type); + if (token.FileId != null) + { + writer.WritePropertyName("file"); + writer.WriteValue(token.FileId); + } if (token.Line != null) { writer.WritePropertyName("line"); writer.WriteValue(token.Line); } - if (token.Line != null) + if (token.Column != null) { writer.WritePropertyName("col"); writer.WriteValue(token.Column); @@ -301,12 +335,17 @@ namespace GitHub.DistributedTask.ObjectTemplating.Tokens writer.WriteStartObject(); writer.WritePropertyName("type"); writer.WriteValue(token.Type); + if (token.FileId != null) + { + writer.WritePropertyName("file"); + writer.WriteValue(token.FileId); + } if (token.Line != null) { writer.WritePropertyName("line"); writer.WriteValue(token.Line); } - if (token.Line != null) + if (token.Column != null) { writer.WritePropertyName("col"); writer.WriteValue(token.Column); diff --git a/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs b/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs index 130b85293..36a8864cd 100644 --- a/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs +++ b/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs @@ -39,7 +39,8 @@ namespace GitHub.DistributedTask.Pipelines DictionaryContextData contextData, WorkspaceOptions workspaceOptions, IEnumerable steps, - IEnumerable scopes) + IEnumerable scopes, + IList fileTable) { this.MessageType = JobRequestMessageTypes.PipelineAgentJobRequest; this.Plan = plan; @@ -74,6 +75,11 @@ namespace GitHub.DistributedTask.Pipelines this.ContextData[pair.Key] = pair.Value; } } + + if (fileTable?.Count > 0) + { + m_fileTable = new List(fileTable); + } } [DataMember] @@ -237,6 +243,18 @@ namespace GitHub.DistributedTask.Pipelines } } + public IList FileTable + { + get + { + if (m_fileTable == null) + { + m_fileTable = new List(); + } + return m_fileTable; + } + } + // todo: remove after feature-flag DistributedTask.EvaluateContainerOnRunner is enabled everywhere public void SetJobSidecarContainers(IDictionary value) { @@ -345,6 +363,11 @@ namespace GitHub.DistributedTask.Pipelines m_environmentVariables = null; } + if (m_fileTable?.Count == 0) + { + m_fileTable = null; + } + if (m_maskHints?.Count == 0) { m_maskHints = null; @@ -374,6 +397,9 @@ namespace GitHub.DistributedTask.Pipelines [DataMember(Name = "EnvironmentVariables", EmitDefaultValue = false)] private List m_environmentVariables; + [DataMember(Name = "FileTable", EmitDefaultValue = false)] + private List m_fileTable; + [DataMember(Name = "Mask", EmitDefaultValue = false)] private List m_maskHints; diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs index de9e7bc65..1d10a3adc 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs @@ -19,7 +19,8 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating { public PipelineTemplateEvaluator( ITraceWriter trace, - TemplateSchema schema) + TemplateSchema schema, + IList fileTable) { if (!String.Equals(schema.Version, PipelineTemplateConstants.Workflow_1_0, StringComparison.Ordinal)) { @@ -28,6 +29,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating m_trace = trace; m_schema = schema; + m_fileTable = fileTable; } public Int32 MaxDepth => 50; @@ -324,6 +326,16 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating TraceWriter = m_trace, }; + // Add the file table + if (m_fileTable?.Count > 0) + { + foreach (var file in m_fileTable) + { + result.GetFileId(file); + } + } + + // Add named context if (contextData != null) { foreach (var pair in contextData) @@ -346,6 +358,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating private readonly ITraceWriter m_trace; private readonly TemplateSchema m_schema; + private readonly IList m_fileTable; private readonly String[] s_contextNames = new[] { PipelineTemplateConstants.GitHub, diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/YamlObjectReader.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/YamlObjectReader.cs new file mode 100644 index 000000000..881b70ae2 --- /dev/null +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/YamlObjectReader.cs @@ -0,0 +1,572 @@ +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using GitHub.DistributedTask.ObjectTemplating; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; + +namespace GitHub.DistributedTask.Pipelines.ObjectTemplating +{ + /// + /// Converts a YAML file into a TemplateToken + /// + public sealed class YamlObjectReader : IObjectReader + { + internal YamlObjectReader( + Int32? fileId, + TextReader input) + { + m_fileId = fileId; + m_parser = new Parser(input); + } + + public Boolean AllowLiteral(out LiteralToken value) + { + if (EvaluateCurrent() is Scalar scalar) + { + // Tag specified + if (!String.IsNullOrEmpty(scalar.Tag)) + { + // String tag + if (String.Equals(scalar.Tag, c_stringTag, StringComparison.Ordinal)) + { + value = new StringToken(m_fileId, scalar.Start.Line, scalar.Start.Column, scalar.Value); + MoveNext(); + return true; + } + + // Not plain style + if (scalar.Style != ScalarStyle.Plain) + { + throw new NotSupportedException($"The scalar style '{scalar.Style}' on line {scalar.Start.Line} and column {scalar.Start.Column} is not valid with the tag '{scalar.Tag}'"); + } + + // Boolean, Float, Integer, or Null + switch (scalar.Tag) + { + case c_booleanTag: + value = ParseBoolean(scalar); + break; + case c_floatTag: + value = ParseFloat(scalar); + break; + case c_integerTag: + value = ParseInteger(scalar); + break; + case c_nullTag: + value = ParseNull(scalar); + break; + default: + throw new NotSupportedException($"Unexpected tag '{scalar.Tag}'"); + } + + MoveNext(); + return true; + } + + // Plain style, determine type using YAML 1.2 "core" schema https://yaml.org/spec/1.2/spec.html#id2804923 + if (scalar.Style == ScalarStyle.Plain) + { + if (MatchNull(scalar, out var nullToken)) + { + value = nullToken; + } + else if (MatchBoolean(scalar, out var booleanToken)) + { + value = booleanToken; + } + else if (MatchInteger(scalar, out var numberToken) || + MatchFloat(scalar, out numberToken)) + { + value = numberToken; + } + else + { + value = new StringToken(m_fileId, scalar.Start.Line, scalar.Start.Column, scalar.Value); + } + + MoveNext(); + return true; + } + + // Otherwise assume string + value = new StringToken(m_fileId, scalar.Start.Line, scalar.Start.Column, scalar.Value); + MoveNext(); + return true; + } + + value = default; + return false; + } + + public Boolean AllowSequenceStart(out SequenceToken value) + { + if (EvaluateCurrent() is SequenceStart sequenceStart) + { + value = new SequenceToken(m_fileId, sequenceStart.Start.Line, sequenceStart.Start.Column); + MoveNext(); + return true; + } + + value = default; + return false; + } + + public Boolean AllowSequenceEnd() + { + if (EvaluateCurrent() is SequenceEnd) + { + MoveNext(); + return true; + } + + return false; + } + + public Boolean AllowMappingStart(out MappingToken value) + { + if (EvaluateCurrent() is MappingStart mappingStart) + { + value = new MappingToken(m_fileId, mappingStart.Start.Line, mappingStart.Start.Column); + MoveNext(); + return true; + } + + value = default; + return false; + } + + public Boolean AllowMappingEnd() + { + if (EvaluateCurrent() is MappingEnd) + { + MoveNext(); + return true; + } + + return false; + } + + /// + /// Consumes the last parsing events, which are expected to be DocumentEnd and StreamEnd. + /// + public void ValidateEnd() + { + if (EvaluateCurrent() is DocumentEnd) + { + MoveNext(); + } + else + { + throw new InvalidOperationException("Expected document end parse event"); + } + + if (EvaluateCurrent() is StreamEnd) + { + MoveNext(); + } + else + { + throw new InvalidOperationException("Expected stream end parse event"); + } + + if (MoveNext()) + { + throw new InvalidOperationException("Expected end of parse events"); + } + } + + /// + /// Consumes the first parsing events, which are expected to be StreamStart and DocumentStart. + /// + public void ValidateStart() + { + if (EvaluateCurrent() != null) + { + throw new InvalidOperationException("Unexpected parser state"); + } + + if (!MoveNext()) + { + throw new InvalidOperationException("Expected a parse event"); + } + + if (EvaluateCurrent() is StreamStart) + { + MoveNext(); + } + else + { + throw new InvalidOperationException("Expected stream start parse event"); + } + + if (EvaluateCurrent() is DocumentStart) + { + MoveNext(); + } + else + { + throw new InvalidOperationException("Expected document start parse event"); + } + } + + private ParsingEvent EvaluateCurrent() + { + if (m_current == null) + { + m_current = m_parser.Current; + if (m_current != null) + { + if (m_current is Scalar scalar) + { + // Verify not using achors + if (scalar.Anchor != null) + { + throw new InvalidOperationException($"Anchors are not currently supported. Remove the anchor '{scalar.Anchor}'"); + } + } + else if (m_current is MappingStart mappingStart) + { + // Verify not using achors + if (mappingStart.Anchor != null) + { + throw new InvalidOperationException($"Anchors are not currently supported. Remove the anchor '{mappingStart.Anchor}'"); + } + } + else if (m_current is SequenceStart sequenceStart) + { + // Verify not using achors + if (sequenceStart.Anchor != null) + { + throw new InvalidOperationException($"Anchors are not currently supported. Remove the anchor '{sequenceStart.Anchor}'"); + } + } + else if (!(m_current is MappingEnd) && + !(m_current is SequenceEnd) && + !(m_current is DocumentStart) && + !(m_current is DocumentEnd) && + !(m_current is StreamStart) && + !(m_current is StreamEnd)) + { + throw new InvalidOperationException($"Unexpected parsing event type: {m_current.GetType().Name}"); + } + } + } + + return m_current; + } + + private Boolean MoveNext() + { + m_current = null; + return m_parser.MoveNext(); + } + + private BooleanToken ParseBoolean(Scalar scalar) + { + if (MatchBoolean(scalar, out var token)) + { + return token; + } + + ThrowInvalidValue(scalar, c_booleanTag); // throws + return default; + } + + private NumberToken ParseFloat(Scalar scalar) + { + if (MatchFloat(scalar, out var token)) + { + return token; + } + + ThrowInvalidValue(scalar, c_floatTag); // throws + return default; + } + + private NumberToken ParseInteger(Scalar scalar) + { + if (MatchInteger(scalar, out var token)) + { + return token; + } + + ThrowInvalidValue(scalar, c_integerTag); // throws + return default; + } + + private NullToken ParseNull(Scalar scalar) + { + if (MatchNull(scalar, out var token)) + { + return token; + } + + ThrowInvalidValue(scalar, c_nullTag); // throws + return default; + } + + private Boolean MatchBoolean( + Scalar scalar, + out BooleanToken value) + { + // YAML 1.2 "core" schema https://yaml.org/spec/1.2/spec.html#id2804923 + switch (scalar.Value ?? String.Empty) + { + case "true": + case "True": + case "TRUE": + value = new BooleanToken(m_fileId, scalar.Start.Line, scalar.Start.Column, true); + return true; + case "false": + case "False": + case "FALSE": + value = new BooleanToken(m_fileId, scalar.Start.Line, scalar.Start.Column, false); + return true; + } + + value = default; + return false; + } + + private Boolean MatchFloat( + Scalar scalar, + out NumberToken value) + { + // YAML 1.2 "core" schema https://yaml.org/spec/1.2/spec.html#id2804923 + var str = scalar.Value; + if (!String.IsNullOrEmpty(str)) + { + // Check for [-+]?(\.inf|\.Inf|\.INF)|\.nan|\.NaN|\.NAN + switch (str) + { + case ".inf": + case ".Inf": + case ".INF": + case "+.inf": + case "+.Inf": + case "+.INF": + value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, Double.PositiveInfinity); + return true; + case "-.inf": + case "-.Inf": + case "-.INF": + value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, Double.NegativeInfinity); + return true; + case ".nan": + case ".NaN": + case ".NAN": + value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, Double.NaN); + return true; + } + + // Otherwise check [-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)? + + // Skip leading sign + var index = str[0] == '-' || str[0] == '+' ? 1 : 0; + + // Check for integer portion + var length = str.Length; + var hasInteger = false; + while (index < length && str[index] >= '0' && str[index] <= '9') + { + hasInteger = true; + index++; + } + + // Check for decimal point + var hasDot = false; + if (index < length && str[index] == '.') + { + hasDot = true; + index++; + } + + // Check for decimal portion + var hasDecimal = false; + while (index < length && str[index] >= '0' && str[index] <= '9') + { + hasDecimal = true; + index++; + } + + // Check [-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?) + if ((hasDot && hasDecimal) || hasInteger) + { + // Check for end + if (index == length) + { + // Try parse + if (Double.TryParse(str, NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out var doubleValue)) + { + value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, doubleValue); + return true; + } + // Otherwise exceeds range + else + { + ThrowInvalidValue(scalar, c_floatTag); // throws + } + } + // Check [eE][-+]?[0-9] + else if (index < length && (str[index] == 'e' || str[index] == 'E')) + { + index++; + + // Skip sign + if (index < length && (str[index] == '-' || str[index] == '+')) + { + index++; + } + + // Check for exponent + var hasExponent = false; + while (index < length && str[index] >= '0' && str[index] <= '9') + { + hasExponent = true; + index++; + } + + // Check for end + if (hasExponent && index == length) + { + // Try parse + if (Double.TryParse(str, NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint | NumberStyles.AllowExponent, CultureInfo.InvariantCulture, out var doubleValue)) + { + value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, (Double)doubleValue); + return true; + } + // Otherwise exceeds range + else + { + ThrowInvalidValue(scalar, c_floatTag); // throws + } + } + } + } + } + + value = default; + return false; + } + + private Boolean MatchInteger( + Scalar scalar, + out NumberToken value) + { + // YAML 1.2 "core" schema https://yaml.org/spec/1.2/spec.html#id2804923 + var str = scalar.Value; + if (!String.IsNullOrEmpty(str)) + { + // Check for [0-9]+ + var firstChar = str[0]; + if (firstChar >= '0' && firstChar <= '9' && + str.Skip(1).All(x => x >= '0' && x <= '9')) + { + // Try parse + if (Double.TryParse(str, NumberStyles.None, CultureInfo.InvariantCulture, out var doubleValue)) + { + value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, doubleValue); + return true; + } + + // Otherwise exceeds range + ThrowInvalidValue(scalar, c_integerTag); // throws + } + // Check for (-|+)[0-9]+ + else if ((firstChar == '-' || firstChar == '+') && + str.Length > 1 && + str.Skip(1).All(x => x >= '0' && x <= '9')) + { + // Try parse + if (Double.TryParse(str, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out var doubleValue)) + { + value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, doubleValue); + return true; + } + + // Otherwise exceeds range + ThrowInvalidValue(scalar, c_integerTag); // throws + } + // Check for 0x[0-9a-fA-F]+ + else if (firstChar == '0' && + str.Length > 2 && + str[1] == 'x' && + str.Skip(2).All(x => (x >= '0' && x <= '9') || (x >= 'a' && x <= 'f') || (x >= 'A' && x <= 'F'))) + { + // Try parse + if (Int32.TryParse(str.Substring(2), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out var integerValue)) + { + value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, integerValue); + return true; + } + + // Otherwise exceeds range + ThrowInvalidValue(scalar, c_integerTag); // throws + } + // Check for 0o[0-9]+ + else if (firstChar == '0' && + str.Length > 2 && + str[1] == 'o' && + str.Skip(2).All(x => x >= '0' && x <= '7')) + { + // Try parse + var integerValue = default(Int32); + try + { + integerValue = Convert.ToInt32(str.Substring(2), 8); + } + // Otherwise exceeds range + catch (Exception) + { + ThrowInvalidValue(scalar, c_integerTag); // throws + } + + value = new NumberToken(m_fileId, scalar.Start.Line, scalar.Start.Column, integerValue); + return true; + } + } + + value = default; + return false; + } + + private Boolean MatchNull( + Scalar scalar, + out NullToken value) + { + // YAML 1.2 "core" schema https://yaml.org/spec/1.2/spec.html#id2804923 + switch (scalar.Value ?? String.Empty) + { + case "": + case "null": + case "Null": + case "NULL": + case "~": + value = new NullToken(m_fileId, scalar.Start.Line, scalar.Start.Column); + return true; + } + + value = default; + return false; + } + + private void ThrowInvalidValue( + Scalar scalar, + String tag) + { + throw new NotSupportedException($"The value '{scalar.Value}' on line {scalar.Start.Line} and column {scalar.Start.Column} is invalid for the type '{scalar.Tag}'"); + } + + private const String c_booleanTag = "tag:yaml.org,2002:bool"; + private const String c_floatTag = "tag:yaml.org,2002:float"; + private const String c_integerTag = "tag:yaml.org,2002:int"; + private const String c_nullTag = "tag:yaml.org,2002:null"; + private const String c_stringTag = "tag:yaml.org,2002:string"; + private readonly Int32? m_fileId; + private readonly Parser m_parser; + private ParsingEvent m_current; + } +} \ No newline at end of file diff --git a/src/Test/L0/Listener/JobDispatcherL0.cs b/src/Test/L0/Listener/JobDispatcherL0.cs index 5f774c174..b81606d54 100644 --- a/src/Test/L0/Listener/JobDispatcherL0.cs +++ b/src/Test/L0/Listener/JobDispatcherL0.cs @@ -33,7 +33,7 @@ namespace GitHub.Runner.Common.Tests.Listener TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = null; Guid jobId = Guid.NewGuid(); - var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null); + var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); result.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); return result; } diff --git a/src/Test/L0/Listener/RunnerL0.cs b/src/Test/L0/Listener/RunnerL0.cs index 8548a035d..07e80e9ce 100644 --- a/src/Test/L0/Listener/RunnerL0.cs +++ b/src/Test/L0/Listener/RunnerL0.cs @@ -43,7 +43,7 @@ namespace GitHub.Runner.Common.Tests.Listener TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = null; Guid jobId = Guid.NewGuid(); - return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null); + return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); } private JobCancelMessage CreateJobCancelMessage() diff --git a/src/Test/L0/Worker/ActionCommandManagerL0.cs b/src/Test/L0/Worker/ActionCommandManagerL0.cs index a1f69c92d..a6cdc086e 100644 --- a/src/Test/L0/Worker/ActionCommandManagerL0.cs +++ b/src/Test/L0/Worker/ActionCommandManagerL0.cs @@ -150,7 +150,7 @@ namespace GitHub.Runner.Common.Tests.Worker TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 6293a3bef..513d286cc 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -25,7 +25,7 @@ namespace GitHub.Runner.Common.Tests.Worker TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -101,7 +101,7 @@ namespace GitHub.Runner.Common.Tests.Worker TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, @@ -152,7 +152,7 @@ namespace GitHub.Runner.Common.Tests.Worker TimelineReference timeline = new TimelineReference(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() { Alias = Pipelines.PipelineConstants.SelfAlias, diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index 447a760dc..209f915e9 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -100,7 +100,7 @@ namespace GitHub.Runner.Common.Tests.Worker }; Guid jobId = Guid.NewGuid(); - _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null); + _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null); GitHubContext github = new GitHubContext(); github["repository"] = new Pipelines.ContextData.StringContextData("actions/runner"); _message.ContextData.Add("github", github); diff --git a/src/Test/L0/Worker/JobRunnerL0.cs b/src/Test/L0/Worker/JobRunnerL0.cs index a87c01987..a88e6b8a7 100644 --- a/src/Test/L0/Worker/JobRunnerL0.cs +++ b/src/Test/L0/Worker/JobRunnerL0.cs @@ -63,7 +63,7 @@ namespace GitHub.Runner.Common.Tests.Worker TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); TimelineReference timeline = new Timeline(Guid.NewGuid()); Guid jobId = Guid.NewGuid(); - _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, testName, testName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null); + _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, testName, testName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null); _message.Variables[Constants.Variables.System.Culture] = "en-US"; _message.Resources.Endpoints.Add(new ServiceEndpoint() { diff --git a/src/Test/L0/Worker/WorkerL0.cs b/src/Test/L0/Worker/WorkerL0.cs index bd8989bea..b48542e79 100644 --- a/src/Test/L0/Worker/WorkerL0.cs +++ b/src/Test/L0/Worker/WorkerL0.cs @@ -67,7 +67,7 @@ namespace GitHub.Runner.Common.Tests.Worker new Pipelines.ContextData.DictionaryContextData() }, }; - var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List(), resources, context, null, actions, null); + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, JobId, jobName, jobName, new StringToken(null, null, null, "ubuntu"), sidecarContainers, null, variables, new List(), resources, context, null, actions, null, null); return jobRequest; }