Compare commits

..

3 Commits

Author SHA1 Message Date
TingluoHuang
edfad1a198 c 2020-05-11 14:28:43 -04:00
TingluoHuang
10bce3549c windows 2020-05-11 14:23:15 -04:00
TingluoHuang
68b180a229 run codeql on all platforms. 2020-05-11 14:09:58 -04:00
18 changed files with 135 additions and 202 deletions

View File

@@ -9,11 +9,11 @@ jobs:
CodeQL-Build:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
# CodeQL runs on ubuntu-latest, windows-latest, and macos-latest
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
steps:
- name: Checkout repository
@@ -22,13 +22,17 @@ jobs:
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
# Override language selection by uncommenting this and choosing your languages
# with:
# languages: go, javascript, csharp, python, cpp, java
- name: Manual build
run : |
./dev.sh layout Release linux-x64
- name: Manual build (Windows)
if: runner.os == 'windows'
run : |
dev.cmd layout Release
working-directory: src
- name: Manual build (Non-Windows)
if: runner.os != 'windows'
run : |
./dev.sh layout Release
working-directory: src
- name: Perform CodeQL Analysis

View File

@@ -1,12 +1,9 @@
## Features
- N/A
- Runner support for GHES Alpha (#381 #386 #390 #393 $401)
- Allow secrets context in Container.env (#388)
## Bugs
- Handle `jq` returns "null" if the field does not exist in create-latest-svc.sh (#478)
- Switch GITHUB_URL to GITHUB_SERVER_URL (#482)
- Fix problem matcher for GHES (#488)
- Fix container action inputs validation warning (#490)
- Fix post step display name (#490)
- Fix worker crash due to exception from evaluating step.env (#490)
- Raise warning when volume mount root. (#413)
- Fix typo (#394)
## Misc
- N/A

View File

@@ -1 +1 @@
2.263.0
2.168.0

View File

@@ -81,7 +81,7 @@ fi
export RUNNER_TOKEN=$(curl -s -X POST ${base_api_url}/${orgs_or_repos}/${runner_scope}/actions/runners/registration-token -H "accept: application/vnd.github.everest-preview+json" -H "authorization: token ${RUNNER_CFG_PAT}" | jq -r '.token')
if [ "null" == "$RUNNER_TOKEN" -o -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi
if [ -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi
#---------------------------------------
# Download latest released and extract

View File

@@ -1,7 +1,6 @@
#!/bin/bash
SVC_NAME="{{SvcNameVar}}"
SVC_NAME=${SVC_NAME// /_}
SVC_DESCRIPTION="{{SvcDescription}}"
user_id=`id -u`

View File

@@ -1,7 +1,6 @@
#!/bin/bash
SVC_NAME="{{SvcNameVar}}"
SVC_NAME=${SVC_NAME// /_}
SVC_DESCRIPTION="{{SvcDescription}}"
SVC_CMD=$1

View File

@@ -92,11 +92,9 @@ namespace GitHub.Runner.Listener.Configuration
_term.WriteSection("Authentication");
while (true)
{
// When testing against a dev deployment of Actions Service, set this environment variable
var useDevActionsServiceUrl = Environment.GetEnvironmentVariable("USE_DEV_ACTIONS_SERVICE_URL");
// Get the URL
var inputUrl = command.GetUrl();
if (inputUrl.Contains("codedev.ms", StringComparison.OrdinalIgnoreCase)
|| useDevActionsServiceUrl != null)
if (inputUrl.Contains("codedev.ms", StringComparison.OrdinalIgnoreCase))
{
runnerSettings.ServerUrl = inputUrl;
// Get the credentials

View File

@@ -82,7 +82,7 @@ namespace GitHub.Runner.Plugins.Repository.v1_0
executionContext.Output($"Syncing repository: {repoFullName}");
// Repository URL
var githubUrl = executionContext.GetGitHubContext("server_url");
var githubUrl = executionContext.GetGitHubContext("url");
var githubUri = new Uri(!string.IsNullOrEmpty(githubUrl) ? githubUrl : "https://github.com");
var portInfo = githubUri.IsDefaultPort ? string.Empty : $":{githubUri.Port}";
Uri repositoryUrl = new Uri($"{githubUri.Scheme}://{githubUri.Host}{portInfo}/{repoFullName}");

View File

@@ -94,13 +94,6 @@ namespace GitHub.Runner.Worker
if (handlerData.HasPost && (Stage == ActionRunStage.Pre || Stage == ActionRunStage.Main))
{
string postDisplayName = $"Post {this.DisplayName}";
if (Stage == ActionRunStage.Pre &&
this.DisplayName.StartsWith("Pre ", StringComparison.OrdinalIgnoreCase))
{
// Trim the leading `Pre ` from the display name.
// Otherwise, we will get `Post Pre xxx` as DisplayName for the Post step.
postDisplayName = $"Post {this.DisplayName.Substring("Pre ".Length)}";
}
var repositoryReference = Action.Reference as RepositoryPathReference;
var pathString = string.IsNullOrEmpty(repositoryReference.Path) ? string.Empty : $"/{repositoryReference.Path}";
var repoString = string.IsNullOrEmpty(repositoryReference.Ref) ? $"{repositoryReference.Name}{pathString}" :
@@ -162,13 +155,6 @@ namespace GitHub.Runner.Worker
}
var validInputs = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (handlerData.ExecutionType == ActionExecutionType.Container)
{
// container action always accept 'entryPoint' and 'args' as inputs
// https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstepswithargs
validInputs.Add("entryPoint");
validInputs.Add("args");
}
// Merge the default inputs from the definition
if (definition.Data?.Inputs != null)
{
@@ -184,15 +170,11 @@ namespace GitHub.Runner.Worker
}
}
// Validate inputs only for actions with action.yml
if (Action.Reference.Type == Pipelines.ActionSourceType.Repository)
foreach (var input in userInputs)
{
foreach (var input in userInputs)
if (!validInputs.Contains(input))
{
if (!validInputs.Contains(input))
{
ExecutionContext.Warning($"Unexpected input '{input}', valid inputs are ['{string.Join("', '", validInputs)}']");
}
ExecutionContext.Warning($"Unexpected input '{input}', valid inputs are ['{string.Join("', '", validInputs)}']");
}
}

View File

@@ -109,7 +109,6 @@ namespace GitHub.Runner.Worker
public sealed class ExecutionContext : RunnerService, IExecutionContext
{
private const int _maxIssueCount = 10;
private const int _throttlingDelayReportThreshold = 10 * 1000; // Don't report throttling with less than 10 seconds delay
private readonly TimelineRecord _record = new TimelineRecord();
private readonly Dictionary<Guid, TimelineRecord> _detailRecords = new Dictionary<Guid, TimelineRecord>();
@@ -336,7 +335,7 @@ namespace GitHub.Runner.Worker
}
// report total delay caused by server throttling.
if (_totalThrottlingDelayInMilliseconds > _throttlingDelayReportThreshold)
if (_totalThrottlingDelayInMilliseconds > 0)
{
this.Warning($"The job has experienced {TimeSpan.FromMilliseconds(_totalThrottlingDelayInMilliseconds).TotalSeconds} seconds total delay caused by server throttling.");
}
@@ -364,11 +363,7 @@ namespace GitHub.Runner.Worker
}
}
if (Root != this)
{
// only dispose TokenSource for step level ExecutionContext
_cancellationTokenSource?.Dispose();
}
_cancellationTokenSource?.Dispose();
_logger.End();
@@ -856,8 +851,7 @@ namespace GitHub.Runner.Worker
{
Interlocked.Add(ref _totalThrottlingDelayInMilliseconds, Convert.ToInt64(data.Delay.TotalMilliseconds));
if (!_throttlingReported &&
_totalThrottlingDelayInMilliseconds > _throttlingDelayReportThreshold)
if (!_throttlingReported)
{
this.Warning(string.Format("The job is currently being throttled by the server. You may experience delays in console line output, job status reporting, and action log uploads."));

View File

@@ -22,8 +22,8 @@ namespace GitHub.Runner.Worker
"repository_owner",
"run_id",
"run_number",
"server_url",
"sha",
"url",
"workflow",
"workspace",
};

View File

@@ -352,24 +352,15 @@ namespace GitHub.Runner.Worker.Handlers
if (File.Exists(gitConfigPath))
{
// Check if the config contains the workflow repository url
var serverUrl = _executionContext.GetGitHubContext("server_url");
serverUrl = !string.IsNullOrEmpty(serverUrl) ? serverUrl : "https://github.com";
var host = new Uri(serverUrl, UriKind.Absolute).Host;
var nameWithOwner = _executionContext.GetGitHubContext("repository");
var patterns = new[] {
$"url = {serverUrl}/{nameWithOwner}",
$"url = git@{host}:{nameWithOwner}.git",
};
var qualifiedRepository = _executionContext.GetGitHubContext("repository");
var configMatch = $"url = https://github.com/{qualifiedRepository}";
var content = File.ReadAllText(gitConfigPath);
foreach (var line in content.Split("\n").Select(x => x.Trim()))
{
foreach (var pattern in patterns)
if (String.Equals(line, configMatch, StringComparison.OrdinalIgnoreCase))
{
if (String.Equals(line, pattern, StringComparison.OrdinalIgnoreCase))
{
repositoryPath = directoryPath;
break;
}
repositoryPath = directoryPath;
break;
}
}
}

View File

@@ -131,13 +131,12 @@ namespace GitHub.Runner.Worker
// Temporary hack for GHES alpha
var configurationStore = HostContext.GetService<IConfigurationStore>();
var runnerSettings = configurationStore.GetSettings();
if (string.IsNullOrEmpty(context.GetGitHubContext("server_url")) && !runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl))
if (string.IsNullOrEmpty(context.GetGitHubContext("url")) && !runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl))
{
var url = new Uri(runnerSettings.GitHubUrl);
var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}";
context.SetGitHubContext("server_url", $"{url.Scheme}://{url.Host}{portInfo}");
context.SetGitHubContext("url", $"{url.Scheme}://{url.Host}{portInfo}");
context.SetGitHubContext("api_url", $"{url.Scheme}://{url.Host}{portInfo}/api/v3");
context.SetGitHubContext("graphql_url", $"{url.Scheme}://{url.Host}{portInfo}/api/graphql");
}
// Evaluate the job-level environment variables

View File

@@ -254,12 +254,6 @@ namespace GitHub.Runner.Worker
Trace.Error(ex);
return TaskResult.Failed;
}
catch (TaskOrchestrationPlanTerminatedException ex)
{
Trace.Error($"TaskOrchestrationPlanTerminatedException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return TaskResult.Failed;
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");

View File

@@ -98,139 +98,124 @@ namespace GitHub.Runner.Worker
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
bool evaluateStepEnvFailed = false;
if (step is IActionRunner actionStep)
{
// Set GITHUB_ACTION
step.ExecutionContext.SetGitHubContext("action", actionStep.Action.Name);
try
// Evaluate and merge action's env block to env context
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator();
var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer);
foreach (var env in actionEnvironment)
{
// Evaluate and merge action's env block to env context
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator();
var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer);
foreach (var env in actionEnvironment)
{
envContext[env.Key] = new StringContextData(env.Value ?? string.Empty);
}
}
catch (Exception ex)
{
// fail the step since there is an evaluate error.
Trace.Info("Caught exception from expression for step.env");
evaluateStepEnvFailed = true;
step.ExecutionContext.Error(ex);
CompleteStep(step, nextStep, TaskResult.Failed);
envContext[env.Key] = new StringContextData(env.Value ?? string.Empty);
}
}
if (!evaluateStepEnvFailed)
try
{
try
// Register job cancellation call back only if job cancellation token not been fire before each step run
if (!jobContext.CancellationToken.IsCancellationRequested)
{
// Register job cancellation call back only if job cancellation token not been fire before each step run
if (!jobContext.CancellationToken.IsCancellationRequested)
// Test the condition again. The job was canceled after the condition was originally evaluated.
jobCancelRegister = jobContext.CancellationToken.Register(() =>
{
// Test the condition again. The job was canceled after the condition was originally evaluated.
jobCancelRegister = jobContext.CancellationToken.Register(() =>
{
// mark job as cancelled
jobContext.Result = TaskResult.Canceled;
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
// mark job as cancelled
jobContext.Result = TaskResult.Canceled;
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'.");
var conditionReTestTraceWriter = new ConditionTraceWriter(Trace, null); // host tracing only
var conditionReTestResult = false;
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
step.ExecutionContext.Debug($"Skip Re-evaluate condition on runner shutdown.");
}
else
{
try
{
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionReTestTraceWriter);
var condition = new BasicExpressionToken(null, null, null, step.Condition);
conditionReTestResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState());
}
catch (Exception ex)
{
// Cancel the step since we get exception while re-evaluate step condition.
Trace.Info("Caught exception from expression when re-test condition on job cancellation.");
step.ExecutionContext.Error(ex);
}
}
if (!conditionReTestResult)
{
// Cancel the step.
Trace.Info("Cancel current running step.");
step.ExecutionContext.CancelToken();
}
});
}
else
{
if (jobContext.Result != TaskResult.Canceled)
step.ExecutionContext.Debug($"Re-evaluate condition on job cancellation for step: '{step.DisplayName}'.");
var conditionReTestTraceWriter = new ConditionTraceWriter(Trace, null); // host tracing only
var conditionReTestResult = false;
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
// mark job as cancelled
jobContext.Result = TaskResult.Canceled;
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
step.ExecutionContext.Debug($"Skip Re-evaluate condition on runner shutdown.");
}
}
// Evaluate condition.
step.ExecutionContext.Debug($"Evaluating condition for step: '{step.DisplayName}'");
var conditionTraceWriter = new ConditionTraceWriter(Trace, step.ExecutionContext);
var conditionResult = false;
var conditionEvaluateError = default(Exception);
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
step.ExecutionContext.Debug($"Skip evaluate condition on runner shutdown.");
}
else
{
try
else
{
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionTraceWriter);
var condition = new BasicExpressionToken(null, null, null, step.Condition);
conditionResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState());
try
{
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionReTestTraceWriter);
var condition = new BasicExpressionToken(null, null, null, step.Condition);
conditionReTestResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState());
}
catch (Exception ex)
{
// Cancel the step since we get exception while re-evaluate step condition.
Trace.Info("Caught exception from expression when re-test condition on job cancellation.");
step.ExecutionContext.Error(ex);
}
}
catch (Exception ex)
{
Trace.Info("Caught exception from expression.");
Trace.Error(ex);
conditionEvaluateError = ex;
}
}
// no evaluate error but condition is false
if (!conditionResult && conditionEvaluateError == null)
if (!conditionReTestResult)
{
// Cancel the step.
Trace.Info("Cancel current running step.");
step.ExecutionContext.CancelToken();
}
});
}
else
{
if (jobContext.Result != TaskResult.Canceled)
{
// Condition == false
Trace.Info("Skipping step due to condition evaluation.");
CompleteStep(step, nextStep, TaskResult.Skipped, resultCode: conditionTraceWriter.Trace);
}
else if (conditionEvaluateError != null)
{
// fail the step since there is an evaluate error.
step.ExecutionContext.Error(conditionEvaluateError);
CompleteStep(step, nextStep, TaskResult.Failed);
}
else
{
// Run the step.
await RunStepAsync(step, jobContext.CancellationToken);
CompleteStep(step, nextStep);
// mark job as cancelled
jobContext.Result = TaskResult.Canceled;
jobContext.JobContext.Status = jobContext.Result?.ToActionResult();
}
}
finally
// Evaluate condition.
step.ExecutionContext.Debug($"Evaluating condition for step: '{step.DisplayName}'");
var conditionTraceWriter = new ConditionTraceWriter(Trace, step.ExecutionContext);
var conditionResult = false;
var conditionEvaluateError = default(Exception);
if (HostContext.RunnerShutdownToken.IsCancellationRequested)
{
if (jobCancelRegister != null)
step.ExecutionContext.Debug($"Skip evaluate condition on runner shutdown.");
}
else
{
try
{
jobCancelRegister?.Dispose();
jobCancelRegister = null;
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionTraceWriter);
var condition = new BasicExpressionToken(null, null, null, step.Condition);
conditionResult = templateEvaluator.EvaluateStepIf(condition, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, step.ExecutionContext.ToExpressionState());
}
catch (Exception ex)
{
Trace.Info("Caught exception from expression.");
Trace.Error(ex);
conditionEvaluateError = ex;
}
}
// no evaluate error but condition is false
if (!conditionResult && conditionEvaluateError == null)
{
// Condition == false
Trace.Info("Skipping step due to condition evaluation.");
CompleteStep(step, nextStep, TaskResult.Skipped, resultCode: conditionTraceWriter.Trace);
}
else if (conditionEvaluateError != null)
{
// fail the step since there is an evaluate error.
step.ExecutionContext.Error(conditionEvaluateError);
CompleteStep(step, nextStep, TaskResult.Failed);
}
else
{
// Run the step.
await RunStepAsync(step, jobContext.CancellationToken);
CompleteStep(step, nextStep);
}
}
finally
{
if (jobCancelRegister != null)
{
jobCancelRegister?.Dispose();
jobCancelRegister = null;
}
}
}

View File

@@ -295,10 +295,9 @@ namespace GitHub.Runner.Common.Tests.Worker
{
Name = "action",
Id = actionId,
Reference = new Pipelines.RepositoryPathReference()
Reference = new Pipelines.ContainerRegistryReference()
{
Name = "actions/runner",
Ref = "v1"
Image = "ubuntu:16.04"
},
Inputs = actionInputs
};

View File

@@ -686,17 +686,14 @@ namespace GitHub.Runner.Common.Tests.Worker
// <WORKSPACE>/workflow-repo/nested-other-repo
// <WORKSPACE>/other-repo
// <WORKSPACE>/other-repo/nested-workflow-repo
// <WORKSPACE>/workflow-repo-using-ssh
var workflowRepository = Path.Combine(workspaceDirectory, "workflow-repo");
var nestedOtherRepository = Path.Combine(workspaceDirectory, "workflow-repo", "nested-other-repo");
var otherRepository = Path.Combine(workspaceDirectory, workflowRepository, "nested-other-repo");
var nestedWorkflowRepository = Path.Combine(workspaceDirectory, "other-repo", "nested-workflow-repo");
var workflowRepositoryUsingSsh = Path.Combine(workspaceDirectory, "workflow-repo-using-ssh");
await CreateRepository(hostContext, workflowRepository, "https://github.com/my-org/workflow-repo");
await CreateRepository(hostContext, nestedOtherRepository, "https://github.com/my-org/other-repo");
await CreateRepository(hostContext, otherRepository, "https://github.com/my-org/other-repo");
await CreateRepository(hostContext, nestedWorkflowRepository, "https://github.com/my-org/workflow-repo");
await CreateRepository(hostContext, workflowRepositoryUsingSsh, "git@github.com:my-org/workflow-repo.git");
// Create test files
var file_noRepository = Path.Combine(workspaceDirectory, "no-repo.txt");
@@ -706,8 +703,7 @@ namespace GitHub.Runner.Common.Tests.Worker
var file_nestedOtherRepository = Path.Combine(nestedOtherRepository, "nested-other-repo");
var file_otherRepository = Path.Combine(otherRepository, "other-repo.txt");
var file_nestedWorkflowRepository = Path.Combine(nestedWorkflowRepository, "nested-workflow-repo.txt");
var file_workflowRepositoryUsingSsh = Path.Combine(workflowRepositoryUsingSsh, "workflow-repo-using-ssh.txt");
foreach (var file in new[] { file_noRepository, file_workflowRepository, file_workflowRepository_nestedDirectory, file_workflowRepository_failsafe, file_nestedOtherRepository, file_otherRepository, file_nestedWorkflowRepository, file_workflowRepositoryUsingSsh })
foreach (var file in new[] { file_noRepository, file_workflowRepository, file_workflowRepository_nestedDirectory, file_workflowRepository_failsafe, file_nestedOtherRepository, file_otherRepository, file_nestedWorkflowRepository })
{
Directory.CreateDirectory(Path.GetDirectoryName(file));
File.WriteAllText(file, "");
@@ -722,9 +718,8 @@ namespace GitHub.Runner.Common.Tests.Worker
Process($"{file_nestedOtherRepository}: some error 6");
Process($"{file_otherRepository}: some error 7");
Process($"{file_nestedWorkflowRepository}: some error 8");
Process($"{file_workflowRepositoryUsingSsh}: some error 9");
Assert.Equal(9, _issues.Count);
Assert.Equal(8, _issues.Count);
Assert.Equal("some error 1", _issues[0].Item1.Message);
Assert.False(_issues[0].Item1.Data.ContainsKey("file"));
@@ -749,9 +744,6 @@ namespace GitHub.Runner.Common.Tests.Worker
Assert.Equal("some error 8", _issues[7].Item1.Message);
Assert.Equal(file_nestedWorkflowRepository.Substring(nestedWorkflowRepository.Length + 1).Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), _issues[7].Item1.Data["file"]);
Assert.Equal("some error 9", _issues[8].Item1.Message);
Assert.Equal(file_workflowRepositoryUsingSsh.Substring(workflowRepositoryUsingSsh.Length + 1).Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), _issues[8].Item1.Data["file"]);
}
Environment.SetEnvironmentVariable("RUNNER_TEST_GET_REPOSITORY_PATH_FAILSAFE", "");

View File

@@ -1 +1 @@
2.263.0
2.169.0