Compare commits

...

15 Commits

Author SHA1 Message Date
Ava S
a1243e21eb Updated node 16 version 2022-12-22 16:13:25 +02:00
Ava S
acdae6d48b Updated sdk version to the latest stable one 2022-12-22 16:12:53 +02:00
Nikola Jokic
332b97f838 Treat jitconfig as secret. (#2335)
Co-authored-by: TingluoHuang <TingluoHuang@github.com>
2022-12-21 13:30:22 -05:00
Stefan Ruvceski
72830cfc12 split by regex (#2333)
* split by regex

* pr fix

* adding tests

* test fix
2022-12-20 14:28:29 +01:00
Tingluo Huang
29a28a870f Make runner image print diag log to STDOUT. (#2331) 2022-12-19 16:57:16 -05:00
Tingluo Huang
0dd7a113f1 Log GitHub RequestId for better traceability. (#2332) 2022-12-19 19:46:29 +00:00
TingluoHuang
83b8baa45e Bump runner version to 2.300.2 to match released version. 2022-12-19 14:22:51 -05:00
Ferenc Hammerl
d5e566ad17 Release notes for 2.300.1 (#2326)
* Update runnerversion

* Update releaseNote.md
2022-12-19 13:08:36 -05:00
Bethany
64381cca6a Re-add file size check prior to reading file (#2321) (#2330)
* Re-add file size check prior to reading file

* Remove redundant file size check
2022-12-19 12:56:28 -05:00
Stefan Ruvceski
f1b1532f32 set env in ProcessInvoker sanitized (#2280)
* set env in ProcessInvoker sanitized
2022-12-19 15:01:53 +01:00
Nikola Jokic
04761e5353 Initialize container manager based on whether the ContainerHooksPath is set (#2317)
* Added tests around checking if correct manager's Initialize method has been called

* repaired missing initialization on container action handler
2022-12-16 15:40:49 +01:00
Ava Stancu
f9e2fa939c Updated contact links for feature requests (#2314)
Users need to use the Github Community feedback page for all feature/enhancement requests.
2022-12-15 17:17:41 +02:00
Ferenc Hammerl
92acb625fb Update Dockerfile (#2315) 2022-12-15 15:44:07 +01:00
Ava Stancu
6b9e8a6be4 prepare release notes for 2.300.0 (#2312)
* Update runner version

* Update releaseNote.md

* Update releaseNote.md

* Update releaseNote.md

Co-authored-by: JoannaaKL <joannaakl@github.com>

* Update releaseNote.md

* Update releaseNote.md

Co-authored-by: JoannaaKL <joannaakl@github.com>
Co-authored-by: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com>
2022-12-14 10:33:02 +02:00
Brittany Ellich
f41f5d259d Use results for uploading step summaries (#2301)
* Use results service for uploading step summaries

* Use results summary over generic results naming convention

* Apply suggestions from code review

Co-authored-by: Tingluo Huang <tingluohuang@github.com>

* Addressing feedback

* Fix merge issue

* Remove empty line

* Update Results json objects to use snake case

* Adding the reference

Co-authored-by: Yang Cao <yacaovsnc@github.com>
Co-authored-by: Tingluo Huang <tingluohuang@github.com>
2022-12-14 09:28:33 +01:00
23 changed files with 652 additions and 79 deletions

View File

@@ -5,7 +5,7 @@
"features": {
"ghcr.io/devcontainers/features/docker-in-docker:1": {},
"ghcr.io/devcontainers/features/dotnet": {
"version": "6.0.300"
"version": "6.0.404"
}
},
"customizations": {

View File

@@ -1,5 +1,8 @@
blank_issues_enabled: false
contact_links:
- name: 🛑 Request a feature in the runner application
url: https://github.com/orgs/community/discussions/categories/actions-and-packages
about: If you have feature requests for GitHub Actions, please use the Actions and Packages section on the Github Product Feedback page.
- name: ✅ Support for GitHub Actions
url: https://github.community/c/code-to-cloud/52
about: If you have questions about GitHub Actions or need support writing workflows, please ask in the GitHub Community Support forum.

View File

@@ -1,32 +0,0 @@
---
name: 🛑 Request a feature in the runner application
about: If you have feature requests for GitHub Actions, please use the "feedback and suggestions for GitHub Actions" link below.
title: ''
labels: enhancement
assignees: ''
---
<!--
👋 You're opening a request for an enhancement in the GitHub Actions **runner application**.
🛑 Please stop if you're not certain that the feature you want is in the runner application - if you have a suggestion for improving GitHub Actions, please see the [GitHub Actions Feedback](https://github.com/github/feedback/discussions/categories/actions-and-packages-feedback) discussion forum which is actively monitored. Using the forum ensures that we route your problem to the correct team. 😃
Some additional useful links:
* If you have found a security issue [please submit it here](https://hackerone.com/github)
* If you have questions or issues with the service, writing workflows or actions, then please [visit the GitHub Community Forum's Actions Board](https://github.community/t5/GitHub-Actions/bd-p/actions)
* If you are having an issue or have a question about GitHub Actions then please [contact customer support](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/about-github-actions#contacting-support)
If you have a feature request that is relevant to this repository, the runner, then please include the information below:
-->
**Describe the enhancement**
A clear and concise description of what the features or enhancement you need.
**Code Snippet**
If applicable, add a code snippet.
**Additional information**
Add any other context about the feature here.
NOTE: if the feature request has been agreed upon then the assignee will create an ADR. See docs/adrs/README.md

View File

@@ -2,7 +2,7 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 as build
ARG RUNNER_VERSION
ARG RUNNER_ARCH="x64"
ARG RUNNER_CONTAINER_HOOKS_VERSION=0.1.3
ARG RUNNER_CONTAINER_HOOKS_VERSION=0.2.0
RUN apt update -y && apt install curl unzip -y
@@ -19,6 +19,7 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0
ENV RUNNER_ALLOW_RUNASROOT=1
ENV RUNNER_MANUALLY_TRAP_SIG=1
ENV ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT=1
WORKDIR /actions-runner
COPY --from=build /actions-runner .

View File

@@ -1,18 +1,17 @@
## Features
- Displays the error logs in dedicated sub-sections of the Initialize containers section (#2182)
- Add generateServiceConfig option for configure command (#2226)
- Setting debug using GitHub Action variables (#2234)
- run.sh installs SIGINT and SIGTERM traps to gracefully stop runner (#2233, #2240)
- Expose github.actor_id, github.workflow_ref & github.workflow_sha as environment variable (#2249)
- Added worker and listener logs to stdout (#2291, #2307)
## Bugs
- Use Global.Variables instead of JobContext and include action path/ref in the message. (#2214)
- Made github.action_status output lowercase to be consistent with job.status' output (#1944)
## Misc
- Allow '--disableupdate' in create-latest-svc.sh (#2201)
- Fix markup for support link (#2114)
- Add runner devcontainer (#2187)
- Setup linter for Runner (#2211, #2213, #2216)
- Added small size runner image for ARC (#2250)
- Small change to Node.js 12 deprecation message (#2262)
- Added the option to use the --replace argument to the create-latest-svc.sh (#2273)
- Made runner_name optional defaulting to hostname in delete.sh script (#1871)
- Return exit code when MANUALLY_TRAP_SIG is exported (#2285)
- Use results for uploading step summaries (#2301) with limited size (#2321)
## Windows x64
We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.

View File

@@ -5,7 +5,7 @@ PRECACHE=$2
NODE_URL=https://nodejs.org/dist
UNOFFICIAL_NODE_URL=https://unofficial-builds.nodejs.org/download/release
NODE12_VERSION="12.22.7"
NODE16_VERSION="16.13.0"
NODE16_VERSION="16.16.0"
get_abs_path() {
# exploits the fact that pwd will print abs path when no args

View File

@@ -90,7 +90,6 @@ namespace GitHub.Runner.Common
public static class Args
{
public static readonly string Auth = "auth";
public static readonly string JitConfig = "jitconfig";
public static readonly string Labels = "labels";
public static readonly string MonitorSocketAddress = "monitorsocketaddress";
public static readonly string Name = "name";
@@ -105,11 +104,13 @@ namespace GitHub.Runner.Common
public static readonly string Token = "token";
public static readonly string PAT = "pat";
public static readonly string WindowsLogonPassword = "windowslogonpassword";
public static readonly string JitConfig = "jitconfig";
public static string[] Secrets => new[]
{
PAT,
Token,
WindowsLogonPassword,
JitConfig,
};
}
@@ -164,6 +165,7 @@ namespace GitHub.Runner.Common
public static readonly string UnsupportedCommandMessageDisabled = "The `{0}` command is disabled. Please upgrade to using Environment Files or opt into unsecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_COMMANDS` environment variable to `true`. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/";
public static readonly string UnsupportedStopCommandTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`.";
public static readonly string UnsupportedSummarySize = "$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of {0}k, got {1}k. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary";
public static readonly string SummaryUploadError = "$GITHUB_STEP_SUMMARY upload aborted, an error occurred when uploading the summary. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary";
public static readonly string Node12DetectedAfterEndOfLife = "Node.js 12 actions are deprecated. Please update the following actions to use Node.js 16: {0}. For more information see: https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/.";
}

View File

@@ -13,6 +13,8 @@ using GitHub.Runner.Sdk;
using GitHub.Services.Common;
using GitHub.Services.WebApi;
using GitHub.Services.WebApi.Utilities.Internal;
using GitHub.Services.Results.Client;
using GitHub.Services.OAuth;
namespace GitHub.Runner.Common
{
@@ -22,11 +24,13 @@ namespace GitHub.Runner.Common
Task ConnectAsync(VssConnection jobConnection);
void InitializeWebsocketClient(ServiceEndpoint serviceEndpoint);
void InitializeResultsClient(Uri uri, string token);
// logging and console
Task<TaskLog> AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken);
Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long? startLine, CancellationToken cancellationToken);
Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken);
Task CreateStepSymmaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken);
Task<TaskLog> CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken);
Task<Timeline> CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken);
Task<List<TimelineRecord>> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable<TimelineRecord> records, CancellationToken cancellationToken);
@@ -40,6 +44,7 @@ namespace GitHub.Runner.Common
private bool _hasConnection;
private VssConnection _connection;
private TaskHttpClient _taskClient;
private ResultsHttpClient _resultsClient;
private ClientWebSocket _websocketClient;
private ServiceEndpoint _serviceEndpoint;
@@ -143,6 +148,12 @@ namespace GitHub.Runner.Common
InitializeWebsocketClient(TimeSpan.Zero);
}
public void InitializeResultsClient(Uri uri, string token)
{
var httpMessageHandler = HostContext.CreateHttpClientHandler();
this._resultsClient = new ResultsHttpClient(uri, httpMessageHandler, token, disposeHandler: true);
}
public ValueTask DisposeAsync()
{
CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None);
@@ -305,6 +316,16 @@ namespace GitHub.Runner.Common
return _taskClient.CreateAttachmentAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, type, name, uploadStream, cancellationToken: cancellationToken);
}
public Task CreateStepSymmaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken)
{
if (_resultsClient != null)
{
return _resultsClient.UploadStepSummaryAsync(planId, jobId, stepId, file, cancellationToken: cancellationToken);
}
throw new InvalidOperationException("Results client is not initialized.");
}
public Task<TaskLog> CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken)
{
CheckConnection();

View File

@@ -20,6 +20,7 @@ namespace GitHub.Runner.Common
void Start(Pipelines.AgentJobRequestMessage jobRequest);
void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null);
void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource);
void QueueSummaryUpload(Guid timelineId, Guid timelineRecordId, string stepId, string name, string path, bool deleteSource);
void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord);
}
@@ -30,6 +31,7 @@ namespace GitHub.Runner.Common
private static readonly TimeSpan _delayForWebConsoleLineDequeue = TimeSpan.FromMilliseconds(500);
private static readonly TimeSpan _delayForTimelineUpdateDequeue = TimeSpan.FromMilliseconds(500);
private static readonly TimeSpan _delayForFileUploadDequeue = TimeSpan.FromMilliseconds(1000);
private static readonly TimeSpan _delayForSummaryUploadDequeue = TimeSpan.FromMilliseconds(1000);
// Job message information
private Guid _scopeIdentifier;
@@ -44,6 +46,8 @@ namespace GitHub.Runner.Common
// queue for file upload (log file or attachment)
private readonly ConcurrentQueue<UploadFileInfo> _fileUploadQueue = new();
private readonly ConcurrentQueue<SummaryUploadFileInfo> _summaryFileUploadQueue = new();
// queue for timeline or timeline record update (one queue per timeline)
private readonly ConcurrentDictionary<Guid, ConcurrentQueue<TimelineRecord>> _timelineUpdateQueue = new();
@@ -56,6 +60,7 @@ namespace GitHub.Runner.Common
// Task for each queue's dequeue process
private Task _webConsoleLineDequeueTask;
private Task _fileUploadDequeueTask;
private Task _summaryUploadDequeueTask;
private Task _timelineUpdateDequeueTask;
// common
@@ -93,6 +98,20 @@ namespace GitHub.Runner.Common
_jobServer.InitializeWebsocketClient(serviceEndPoint);
// This code is usually wrapped by an instance of IExecutionContext which isn't available here.
jobRequest.Variables.TryGetValue("system.github.results_endpoint", out VariableValue resultsEndpointVariable);
var resultsReceiverEndpoint = resultsEndpointVariable?.Value;
if (serviceEndPoint?.Authorization != null &&
serviceEndPoint.Authorization.Parameters.TryGetValue("AccessToken", out var accessToken) &&
!string.IsNullOrEmpty(accessToken) &&
!string.IsNullOrEmpty(resultsReceiverEndpoint))
{
Trace.Info("Initializing results client");
_jobServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), accessToken);
}
if (_queueInProcess)
{
Trace.Info("No-opt, all queue process tasks are running.");
@@ -120,10 +139,13 @@ namespace GitHub.Runner.Common
Trace.Info("Start process file upload queue.");
_fileUploadDequeueTask = ProcessFilesUploadQueueAsync();
Trace.Info("Start results file upload queue.");
_summaryUploadDequeueTask = ProcessSummaryUploadQueueAsync();
Trace.Info("Start process timeline update queue.");
_timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync();
_allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask };
_allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask, _summaryUploadDequeueTask };
_queueInProcess = true;
}
@@ -154,6 +176,10 @@ namespace GitHub.Runner.Common
await ProcessFilesUploadQueueAsync(runOnce: true);
Trace.Info("File upload queue drained.");
Trace.Verbose("Draining results summary upload queue.");
await ProcessSummaryUploadQueueAsync(runOnce: true);
Trace.Info("Results summary upload queue drained.");
// ProcessTimelinesUpdateQueueAsync() will throw exception during shutdown
// if there is any timeline records that failed to update contains output variabls.
Trace.Verbose("Draining timeline update queue.");
@@ -204,6 +230,28 @@ namespace GitHub.Runner.Common
_fileUploadQueue.Enqueue(newFile);
}
public void QueueSummaryUpload(Guid timelineId, Guid timelineRecordId, string stepId, string name, string path, bool deleteSource)
{
ArgUtil.NotEmpty(timelineId, nameof(timelineId));
ArgUtil.NotEmpty(timelineRecordId, nameof(timelineRecordId));
// all parameter not null, file path exist.
var newFile = new SummaryUploadFileInfo()
{
TimelineId = timelineId,
TimelineRecordId = timelineRecordId,
Name = name,
Path = path,
PlanId = _planId.ToString(),
JobId = _jobTimelineRecordId.ToString(),
StepId = stepId,
DeleteSource = deleteSource
};
Trace.Verbose("Enqueue results file upload queue: file '{0}' attach to record {1}", newFile.Path, timelineRecordId);
_summaryFileUploadQueue.Enqueue(newFile);
}
public void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord)
{
ArgUtil.NotEmpty(timelineId, nameof(timelineId));
@@ -299,7 +347,7 @@ namespace GitHub.Runner.Common
{
try
{
// Give at most 60s for each request.
// Give at most 60s for each request.
using (var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(60)))
{
await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), batch[0].LineNumber, timeoutTokenSource.Token);
@@ -394,6 +442,60 @@ namespace GitHub.Runner.Common
}
}
private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false)
{
Trace.Info("Starting results-based upload queue...");
while (!_jobCompletionSource.Task.IsCompleted || runOnce)
{
List<SummaryUploadFileInfo> filesToUpload = new();
SummaryUploadFileInfo dequeueFile;
while (_summaryFileUploadQueue.TryDequeue(out dequeueFile))
{
filesToUpload.Add(dequeueFile);
// process at most 10 file upload.
if (!runOnce && filesToUpload.Count > 10)
{
break;
}
}
if (filesToUpload.Count > 0)
{
if (runOnce)
{
Trace.Info($"Uploading {filesToUpload.Count} summary files in one shot through results service.");
}
int errorCount = 0;
foreach (var file in filesToUpload)
{
try
{
await UploadSummaryFile(file);
}
catch (Exception ex)
{
Trace.Info("Catch exception during summary file upload to results, keep going since the process is best effort.");
Trace.Error(ex);
errorCount++;
}
}
Trace.Info("Tried to upload {0} summary files to results, success rate: {1}/{0}.", filesToUpload.Count, filesToUpload.Count - errorCount);
}
if (runOnce)
{
break;
}
else
{
await Task.Delay(_delayForSummaryUploadDequeue);
}
}
}
private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false)
{
while (!_jobCompletionSource.Task.IsCompleted || runOnce)
@@ -665,6 +767,35 @@ namespace GitHub.Runner.Common
}
}
}
private async Task UploadSummaryFile(SummaryUploadFileInfo file)
{
bool uploadSucceed = false;
try
{
// Upload the step summary
Trace.Info($"Starting to upload summary file to results service {file.Name}, {file.Path}");
var cancellationTokenSource = new CancellationTokenSource();
await _jobServer.CreateStepSymmaryAsync(file.PlanId, file.JobId, file.StepId, file.Path, cancellationTokenSource.Token);
uploadSucceed = true;
}
finally
{
if (uploadSucceed && file.DeleteSource)
{
try
{
File.Delete(file.Path);
}
catch (Exception ex)
{
Trace.Info("Catch exception during delete success results uploaded summary file.");
Trace.Error(ex);
}
}
}
}
}
internal class PendingTimelineRecord
@@ -683,6 +814,19 @@ namespace GitHub.Runner.Common
public bool DeleteSource { get; set; }
}
internal class SummaryUploadFileInfo
{
public Guid TimelineId { get; set; }
public Guid TimelineRecordId { get; set; }
public string Name { get; set; }
public string Path { get; set; }
public string PlanId { get; set; }
public string JobId { get; set; }
public string StepId { get; set; }
public bool DeleteSource { get; set; }
}
internal class ConsoleLineInfo
{

View File

@@ -1,10 +1,3 @@
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using GitHub.Services.Common;
using GitHub.Services.Common.Internal;
using GitHub.Services.OAuth;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -14,6 +7,13 @@ using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks;
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using GitHub.Services.Common;
using GitHub.Services.Common.Internal;
using GitHub.Services.OAuth;
namespace GitHub.Runner.Listener.Configuration
{
@@ -636,7 +636,7 @@ namespace GitHub.Runner.Listener.Configuration
}
int retryCount = 0;
while(retryCount < 3)
while (retryCount < 3)
{
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
@@ -646,28 +646,29 @@ namespace GitHub.Runner.Listener.Configuration
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("basic", base64EncodingToken);
httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
httpClient.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.github.v3+json");
var responseStatus = System.Net.HttpStatusCode.OK;
try
{
var response = await httpClient.PostAsync(githubApiUrl, new StringContent(string.Empty));
responseStatus = response.StatusCode;
var githubRequestId = GetGitHubRequestId(response.Headers);
if (response.IsSuccessStatusCode)
{
Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}' ({githubRequestId})");
var jsonResponse = await response.Content.ReadAsStringAsync();
return StringUtil.ConvertFromJson<GitHubRunnerRegisterToken>(jsonResponse);
}
else
{
_term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
_term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}' (Request Id: {githubRequestId})");
var errorResponse = await response.Content.ReadAsStringAsync();
_term.WriteError(errorResponse);
response.EnsureSuccessStatusCode();
}
}
catch(Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound)
catch (Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound)
{
retryCount++;
Trace.Error($"Failed to get JIT runner token -- Atempt: {retryCount}");
@@ -714,22 +715,23 @@ namespace GitHub.Runner.Listener.Configuration
{
var response = await httpClient.PostAsync(githubApiUrl, new StringContent(StringUtil.ConvertToJson(bodyObject), null, "application/json"));
responseStatus = response.StatusCode;
var githubRequestId = GetGitHubRequestId(response.Headers);
if(response.IsSuccessStatusCode)
if (response.IsSuccessStatusCode)
{
Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}' ({githubRequestId})");
var jsonResponse = await response.Content.ReadAsStringAsync();
return StringUtil.ConvertFromJson<GitHubAuthResult>(jsonResponse);
}
else
{
_term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
_term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}' (Request Id: {githubRequestId})");
var errorResponse = await response.Content.ReadAsStringAsync();
_term.WriteError(errorResponse);
response.EnsureSuccessStatusCode();
}
}
catch(Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound)
catch (Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound)
{
retryCount++;
Trace.Error($"Failed to get tenant credentials -- Atempt: {retryCount}");
@@ -742,5 +744,14 @@ namespace GitHub.Runner.Listener.Configuration
}
return null;
}
private string GetGitHubRequestId(HttpResponseHeaders headers)
{
if (headers.TryGetValues("x-github-request-id", out var headerValues))
{
return headerValues.FirstOrDefault();
}
return string.Empty;
}
}
}

View File

@@ -11,6 +11,7 @@ using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
using GitHub.Runner.Sdk;
using System.Text.RegularExpressions;
namespace GitHub.Runner.Sdk
{
@@ -264,7 +265,17 @@ namespace GitHub.Runner.Sdk
{
foreach (KeyValuePair<string, string> kvp in environment)
{
#if OS_WINDOWS
string tempKey = String.IsNullOrWhiteSpace(kvp.Key) ? kvp.Key : Regex.Split(kvp.Key, @"\p{C}")[0];
string tempValue = String.IsNullOrWhiteSpace(kvp.Value) ? kvp.Value : Regex.Split(kvp.Value, @"\p{C}")[0];
if(!String.IsNullOrWhiteSpace(tempKey))
{
_proc.StartInfo.Environment[tempKey] = tempValue;
}
#else
_proc.StartInfo.Environment[kvp.Key] = kvp.Value;
#endif
}
}

View File

@@ -33,8 +33,14 @@ namespace GitHub.Runner.Worker
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_dockerManager = HostContext.GetService<IDockerCommandManager>();
_containerHookManager = HostContext.GetService<IContainerHookManager>();
if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(Constants.Hooks.ContainerHooksPath)))
{
_dockerManager = HostContext.GetService<IDockerCommandManager>();
}
else
{
_containerHookManager = HostContext.GetService<IContainerHookManager>();
}
}
public async Task StartContainersAsync(IExecutionContext executionContext, object data)

View File

@@ -80,6 +80,7 @@ namespace GitHub.Runner.Worker
// logging
long Write(string tag, string message);
void QueueAttachFile(string type, string name, string filePath);
void QueueSummaryFile(string name, string filePath, string stepId);
// timeline record update methods
void Start(string currentOperation = null);
@@ -846,6 +847,19 @@ namespace GitHub.Runner.Worker
_jobServerQueue.QueueFileUpload(_mainTimelineId, _record.Id, type, name, filePath, deleteSource: false);
}
public void QueueSummaryFile(string name, string filePath, string stepId)
{
ArgUtil.NotNullOrEmpty(name, nameof(name));
ArgUtil.NotNullOrEmpty(filePath, nameof(filePath));
if (!File.Exists(filePath))
{
throw new FileNotFoundException($"Can't upload (name:{name}) file: {filePath}. File does not exist.");
}
_jobServerQueue.QueueSummaryUpload(_mainTimelineId, _record.Id, stepId, name, filePath, deleteSource: false);
}
// Add OnMatcherChanged
public void Add(OnMatcherChanged handler)
{
@@ -1085,7 +1099,7 @@ namespace GitHub.Runner.Worker
{
if (contextData != null &&
contextData.TryGetValue(PipelineTemplateConstants.Vars, out var varsPipelineContextData) &&
varsPipelineContextData != null &&
varsPipelineContextData != null &&
varsPipelineContextData is DictionaryContextData varsContextData)
{
// Set debug variables only when StepDebug/RunnerDebug variables are not present.

View File

@@ -204,13 +204,24 @@ namespace GitHub.Runner.Worker
}
}
var attachmentName = !context.IsEmbedded
? context.Id.ToString()
var attachmentName = !context.IsEmbedded
? context.Id.ToString()
: context.EmbeddedId.ToString();
Trace.Info($"Queueing file ({filePath}) for attachment upload ({attachmentName})");
// Attachments must be added to the parent context (job), not the current context (step)
context.Root.QueueAttachFile(ChecksAttachmentType.StepSummary, attachmentName, scrubbedFilePath);
context.Global.Variables.TryGetValue("system.github.results_endpoint", out string resultsReceiverEndpoint);
if (resultsReceiverEndpoint != null)
{
Trace.Info($"Queueing results file ({filePath}) for attachment upload ({attachmentName})");
var stepId = context.Id.ToString();
// Attachments must be added to the parent context (job), not the current context (step)
context.Root.QueueSummaryFile(attachmentName, scrubbedFilePath, stepId);
}
else
{
Trace.Info($"Queueing file ({filePath}) for attachment upload ({attachmentName})");
// Attachments must be added to the parent context (job), not the current context (step)
context.Root.QueueAttachFile(ChecksAttachmentType.StepSummary, attachmentName, scrubbedFilePath);
}
}
catch (Exception e)
{

View File

@@ -38,8 +38,17 @@ namespace GitHub.Runner.Worker.Handlers
// Update the env dictionary.
AddInputsToEnvironment();
var dockerManager = HostContext.GetService<IDockerCommandManager>();
var containerHookManager = HostContext.GetService<IContainerHookManager>();
IDockerCommandManager dockerManager = null;
IContainerHookManager containerHookManager = null;
if (FeatureManager.IsContainerHooksEnabled(ExecutionContext.Global.Variables))
{
containerHookManager = HostContext.GetService<IContainerHookManager>();
}
else
{
dockerManager = HostContext.GetService<IDockerCommandManager>();
}
string dockerFile = null;
// container image haven't built/pull

View File

@@ -27,7 +27,6 @@ namespace GitHub.DistributedTask.WebApi
this.Type = type;
this.Name = name;
}
[DataMember]
public String Type

View File

@@ -0,0 +1,60 @@
using System.Runtime.Serialization;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
namespace GitHub.Services.Results.Contracts
{
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class GetSignedStepSummaryURLRequest
{
[DataMember]
public string WorkflowJobRunBackendId;
[DataMember]
public string WorkflowRunBackendId;
[DataMember]
public string StepBackendId;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class GetSignedStepSummaryURLResponse
{
[DataMember]
public string SummaryUrl;
[DataMember]
public long SoftSizeLimit;
[DataMember]
public string BlobStorageType;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class StepSummaryMetadataCreate
{
[DataMember]
public string StepBackendId;
[DataMember]
public string WorkflowRunBackendId;
[DataMember]
public string WorkflowJobRunBackendId;
[DataMember]
public long Size;
[DataMember]
public string UploadedAt;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class CreateStepSummaryMetadataResponse
{
[DataMember]
public bool Ok;
}
public static class BlobStorageTypes
{
public static readonly string AzureBlobStorage = "BLOB_STORAGE_TYPE_AZURE";
public static readonly string Unspecified = "BLOB_STORAGE_TYPE_UNSPECIFIED";
}
}

View File

@@ -0,0 +1,138 @@
using System;
using System.IO;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Services.Results.Contracts;
using System.Net.Http.Formatting;
using Sdk.WebApi.WebApi;
namespace GitHub.Services.Results.Client
{
public class ResultsHttpClient : RawHttpClientBase
{
public ResultsHttpClient(
Uri baseUrl,
HttpMessageHandler pipeline,
string token,
bool disposeHandler)
: base(baseUrl, pipeline, disposeHandler)
{
m_token = token;
m_resultsServiceUrl = baseUrl;
m_formatter = new JsonMediaTypeFormatter();
}
public async Task<GetSignedStepSummaryURLResponse> GetStepSummaryUploadUrlAsync(string planId, string jobId, string stepId, CancellationToken cancellationToken)
{
var request = new GetSignedStepSummaryURLRequest()
{
WorkflowJobRunBackendId= jobId,
WorkflowRunBackendId= planId,
StepBackendId= stepId
};
var stepSummaryUploadRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/GetStepSummarySignedBlobURL");
using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepSummaryUploadRequest))
{
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token);
requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json"));
using (HttpContent content = new ObjectContent<GetSignedStepSummaryURLRequest>(request, m_formatter))
{
requestMessage.Content = content;
using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken))
{
return await ReadJsonContentAsync<GetSignedStepSummaryURLResponse>(response, cancellationToken);
}
}
}
}
private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, string stepId, long size, CancellationToken cancellationToken)
{
var timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK");
var request = new StepSummaryMetadataCreate()
{
WorkflowJobRunBackendId= jobId,
WorkflowRunBackendId= planId,
StepBackendId = stepId,
Size = size,
UploadedAt = timestamp
};
var stepSummaryUploadCompleteRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/CreateStepSummaryMetadata");
using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepSummaryUploadCompleteRequest))
{
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token);
requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json"));
using (HttpContent content = new ObjectContent<StepSummaryMetadataCreate>(request, m_formatter))
{
requestMessage.Content = content;
using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken))
{
var jsonResponse = await ReadJsonContentAsync<CreateStepSummaryMetadataResponse>(response, cancellationToken);
if (!jsonResponse.Ok)
{
throw new Exception($"Failed to mark step summary upload as complete, status code: {response.StatusCode}, ok: {jsonResponse.Ok}, size: {size}, timestamp: {timestamp}");
}
}
}
}
}
private async Task<HttpResponseMessage> UploadFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken)
{
// Upload the file to the url
var request = new HttpRequestMessage(HttpMethod.Put, url)
{
Content = new StreamContent(file)
};
if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
request.Content.Headers.Add("x-ms-blob-type", "BlockBlob");
}
using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
{
if (!response.IsSuccessStatusCode)
{
throw new Exception($"Failed to upload file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}");
}
return response;
}
}
// Handle file upload for step summary
public async Task UploadStepSummaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken)
{
// Get the upload url
var uploadUrlResponse = await GetStepSummaryUploadUrlAsync(planId, jobId, stepId, cancellationToken);
// Do we want to throw an exception here or should we just be uploading/truncating the data
var fileSize = new FileInfo(file).Length;
if (fileSize > uploadUrlResponse.SoftSizeLimit)
{
throw new Exception($"File size is larger than the upload url allows, file size: {fileSize}, upload url size: {uploadUrlResponse.SoftSizeLimit}");
}
// Upload the file
using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
{
var response = await UploadFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken);
}
// Send step summary upload complete message
await StepSummaryUploadCompleteAsync(planId, jobId, stepId, fileSize, cancellationToken);
}
private MediaTypeFormatter m_formatter;
private Uri m_resultsServiceUrl;
private string m_token;
}
}

View File

@@ -128,7 +128,144 @@ namespace GitHub.Runner.Common.Tests
}
}
}
#if OS_WINDOWS
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]
public async Task SetTestEnvWithNullInKey()
{
using (TestHostContext hc = new(this))
{
Tracing trace = hc.GetTrace();
Int32 exitCode = -1;
var processInvoker = new ProcessInvokerWrapper();
processInvoker.Initialize(hc);
var stdout = new List<string>();
var stderr = new List<string>();
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
trace.Info(e.Data);
stdout.Add(e.Data);
};
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
trace.Info(e.Data);
stderr.Add(e.Data);
};
exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %TEST%\"", new Dictionary<string, string>() { { "TEST\0second", "first" } }, CancellationToken.None);
trace.Info("Exit Code: {0}", exitCode);
Assert.Equal(0, exitCode);
Assert.Equal("first", stdout.First(x => !string.IsNullOrWhiteSpace(x)));
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]
public async Task SetTestEnvWithTabInKey()
{
using (TestHostContext hc = new(this))
{
Tracing trace = hc.GetTrace();
Int32 exitCode = -1;
var processInvoker = new ProcessInvokerWrapper();
processInvoker.Initialize(hc);
var stdout = new List<string>();
var stderr = new List<string>();
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
trace.Info(e.Data);
stdout.Add(e.Data);
};
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
trace.Info(e.Data);
stderr.Add(e.Data);
};
exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %TEST%\"", new Dictionary<string, string>() { { "TEST\u0009second", "first" } }, CancellationToken.None);
trace.Info("Exit Code: {0}", exitCode);
Assert.Equal(0, exitCode);
Assert.Equal("first", stdout.First(x => !string.IsNullOrWhiteSpace(x)));
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]
public async Task SetTestEnvWithNullInValue()
{
using (TestHostContext hc = new(this))
{
Tracing trace = hc.GetTrace();
Int32 exitCode = -1;
var processInvoker = new ProcessInvokerWrapper();
processInvoker.Initialize(hc);
var stdout = new List<string>();
var stderr = new List<string>();
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
trace.Info(e.Data);
stdout.Add(e.Data);
};
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
trace.Info(e.Data);
stderr.Add(e.Data);
};
exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %TEST%\"", new Dictionary<string, string>() { { "TEST", "first\0second" } }, CancellationToken.None);
trace.Info("Exit Code: {0}", exitCode);
Assert.Equal(0, exitCode);
Assert.Equal("first", stdout.First(x => !string.IsNullOrWhiteSpace(x)));
}
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]
public async Task SetTestEnvWithTabInValue()
{
using (TestHostContext hc = new(this))
{
Tracing trace = hc.GetTrace();
Int32 exitCode = -1;
var processInvoker = new ProcessInvokerWrapper();
processInvoker.Initialize(hc);
var stdout = new List<string>();
var stderr = new List<string>();
processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
trace.Info(e.Data);
stdout.Add(e.Data);
};
processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) =>
{
trace.Info(e.Data);
stderr.Add(e.Data);
};
exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %TEST%\"", new Dictionary<string, string>() { { "TEST", "first\u0009second" } }, CancellationToken.None);
trace.Info("Exit Code: {0}", exitCode);
Assert.Equal(0, exitCode);
Assert.Equal("first", stdout.First(x => !string.IsNullOrWhiteSpace(x)));
}
}
#endif
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]

View File

@@ -99,6 +99,46 @@ namespace GitHub.Runner.Common.Tests.Worker
}
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Worker")]
public void InitializeWithCorrectManager()
{
containers.Add(new ContainerInfo() { ContainerImage = "ubuntu:16.04" });
_hc = new TestHostContext(this, "Test");
_ec = new Mock<IExecutionContext>();
serverQueue = new Mock<IJobServerQueue>();
pagingLogger = new Mock<IPagingLogger>();
containerOperationProvider = new ContainerOperationProvider();
_hc.SetSingleton<IJobServerQueue>(serverQueue.Object);
_hc.SetSingleton<IPagingLogger>(pagingLogger.Object);
_ec.Setup(x => x.Global).Returns(new GlobalContext());
Environment.SetEnvironmentVariable(Constants.Hooks.ContainerHooksPath, "/tmp/k8s/index.js");
_dockerManager = new Mock<IDockerCommandManager>();
_dockerManager.Setup(x => x.Initialize(_hc)).Throws(new Exception("Docker manager's Initialize should not be called"));
_containerHookManager = new Mock<IContainerHookManager>();
_hc.SetSingleton<IDockerCommandManager>(_dockerManager.Object);
_hc.SetSingleton<IContainerHookManager>(_containerHookManager.Object);
containerOperationProvider.Initialize(_hc);
Environment.SetEnvironmentVariable(Constants.Hooks.ContainerHooksPath, null);
_containerHookManager = new Mock<IContainerHookManager>();
_containerHookManager.Setup(x => x.Initialize(_hc)).Throws(new Exception("Container hook manager's Initialize should not be called"));
_dockerManager = new Mock<IDockerCommandManager>();
_hc.SetSingleton<IDockerCommandManager>(_dockerManager.Object);
_hc.SetSingleton<IContainerHookManager>(_containerHookManager.Object);
containerOperationProvider.Initialize(_hc);
}
private void Setup([CallerMemberName] string testName = "")
{
containers.Add(new ContainerInfo() { ContainerImage = "ubuntu:16.04" });
@@ -111,7 +151,6 @@ namespace GitHub.Runner.Common.Tests.Worker
_containerHookManager = new Mock<IContainerHookManager>();
containerOperationProvider = new ContainerOperationProvider();
_hc.SetSingleton<IDockerCommandManager>(_dockerManager.Object);
_hc.SetSingleton<IJobServerQueue>(serverQueue.Object);
_hc.SetSingleton<IPagingLogger>(pagingLogger.Object);

View File

@@ -22,7 +22,7 @@ DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x"
PACKAGE_DIR="$SCRIPT_DIR/../_package"
PACKAGE_TRIMS_DIR="$SCRIPT_DIR/../_package_trims"
DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk"
DOTNETSDK_VERSION="6.0.300"
DOTNETSDK_VERSION="6.0.404"
DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION"
RUNNER_VERSION=$(cat runnerversion)

View File

@@ -1,5 +1,5 @@
{
"sdk": {
"version": "6.0.300"
"version": "6.0.404"
}
}

View File

@@ -1 +1 @@
2.299.1
2.300.2