GitHub Actions Runner

This commit is contained in:
Tingluo Huang
2019-10-10 00:52:42 -04:00
commit c8afc84840
1255 changed files with 198670 additions and 0 deletions

87
.gitattributes vendored Normal file
View File

@@ -0,0 +1,87 @@
###############################################################################
# Set default behavior to automatically normalize line endings.
###############################################################################
* text=auto
# Shell scripts should always use line feed not crlf
*.sh text eol=lf
###############################################################################
# Set default behavior for command prompt diff.
#
# This is need for earlier builds of msysgit that does not have it on by
# default for csharp files.
# Note: This is only used by command line
###############################################################################
#*.cs diff=csharp
###############################################################################
# Set the merge driver for project and solution files
#
# Merging from the command prompt will add diff markers to the files if there
# are conflicts (Merging from VS is not affected by the settings below, in VS
# the diff markers are never inserted). Diff markers may cause the following
# file extensions to fail to load in VS. An alternative would be to treat
# these files as binary and thus will always conflict and require user
# intervention with every merge. To do so, just uncomment the entries below
###############################################################################
*.js text
*.json text
*.resjson text
*.htm text
*.html text
*.xml text
*.txt text
*.ini text
*.inc text
#*.sln merge=binary
#*.csproj merge=binary
#*.vbproj merge=binary
#*.vcxproj merge=binary
#*.vcproj merge=binary
#*.dbproj merge=binary
#*.fsproj merge=binary
#*.lsproj merge=binary
#*.wixproj merge=binary
#*.modelproj merge=binary
#*.sqlproj merge=binary
#*.wwaproj merge=binary
###############################################################################
# behavior for image files
#
# image files are treated as binary by default.
###############################################################################
*.png binary
*.jpg binary
*.jpeg binary
*.gif binary
*.ico binary
*.mov binary
*.mp4 binary
*.mp3 binary
*.flv binary
*.fla binary
*.swf binary
*.gz binary
*.zip binary
*.7z binary
*.ttf binary
###############################################################################
# diff behavior for common document formats
#
# Convert binary document formats to text before diffing them. This feature
# is only available from the command line. Turn it on by uncommenting the
# entries below.
###############################################################################
*.doc diff=astextplain
*.DOC diff=astextplain
*.docx diff=astextplain
*.DOCX diff=astextplain
*.dot diff=astextplain
*.DOT diff=astextplain
*.pdf diff=astextplain
*.PDF diff=astextplain
*.rtf diff=astextplain
*.RTF diff=astextplain

10
.github/ISSUE_TEMPLATE.md vendored Normal file
View File

@@ -0,0 +1,10 @@
## Runner Version and Platform
Version of your runner?
OS of the machine running the runner? OSX/Windows/Linux/...
## What's not working?
Please include error messages and screenshots.
## Runner and Worker's Diagnostic Logs
Logs are located in the runner's `_diag` folder. The runner logs are prefixed with `Runner_` and the worker logs are prefixed with `Worker_`. All sensitive information should already be masked out, but please double-check before pasting here.

52
.github/workflows/build.yml vendored Normal file
View File

@@ -0,0 +1,52 @@
name: Runner CI
on:
push:
branches:
- master
pull_request:
branches:
- '*'
jobs:
build:
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
include:
- os: ubuntu-latest
devScript: ./dev.sh
- os: macOS-latest
devScript: ./dev.sh
- os: windows-latest
devScript: dev.cmd
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v1
# Build runner layout
- name: Build & Layout Release
run: |
${{ matrix.devScript }} layout Release
working-directory: src
# Run tests
- name: L0
run: |
${{ matrix.devScript }} test
working-directory: src
# Create runner package tar.gz/zip
- name: Package Release
if: github.event_name != 'pull_request'
run: |
${{ matrix.devScript }} package Release
working-directory: src
# Upload runner package tar.gz/zip as artifact
- name: Publish Artifact
if: github.event_name != 'pull_request'
uses: actions/upload-artifact@v1
with:
name: runner-package-${{ matrix.os }}
path: _package

23
.gitignore vendored Normal file
View File

@@ -0,0 +1,23 @@
**/bin
**/obj
**/libs
**/*.xproj
**/*.xproj.user
**/*.sln
**/.vs
**/.vscode
**/*.error
**/*.json.pretty
node_modules
_downloads
_layout
_package
_dotnetsdk
TestResults
TestLogs
.DS_Store
**/*.DotSettings.user
#generated
src/Runner.Sdk/BuildConstants.cs

20
LICENSE Normal file
View File

@@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) Microsoft Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

31
README.md Normal file
View File

@@ -0,0 +1,31 @@
# GitHub Actions Runner
<p align="center">
<img src="docs/res/github-graph.png">
</p>
[![Actions Status](https://github.com/actions/runner/workflows/Runner%20CI/badge.svg)](https://github.com/actions/runner/actions)
## Get Started
![win](docs/res/win_sm.png) [Pre-reqs](docs/start/envwin.md) | [Download](https://github.com/actions/runner/releases/latest)
![macOS](docs/res/apple_sm.png) [Pre-reqs](docs/start/envosx.md) | [Download](https://github.com/actions/runner/releases/latest)
![linux](docs/res/linux_sm.png) [Pre-reqs](docs/start/envlinux.md) | [Download](https://github.com/actions/runner/releases/latest)
**Configure:**
*MacOS and Linux*
```bash
./config.sh
```
*Windows*
```bash
config.cmd
```
## Contribute
For developers that want to contribute, [read here](docs/contribute.md) on how to build and test.

20
assets.json Normal file
View File

@@ -0,0 +1,20 @@
[
{
"name": "actions-runner-win-x64-<RUNNER_VERSION>.zip",
"platform": "win-x64",
"version": "<RUNNER_VERSION>",
"downloadUrl": "https://githubassets.azureedge.net/runners/<RUNNER_VERSION>/actions-runner-win-x64-<RUNNER_VERSION>.zip"
},
{
"name": "actions-runner-osx-x64-<RUNNER_VERSION>.tar.gz",
"platform": "osx-x64",
"version": "<RUNNER_VERSION>",
"downloadUrl": "https://githubassets.azureedge.net/runners/<RUNNER_VERSION>/actions-runner-osx-x64-<RUNNER_VERSION>.tar.gz"
},
{
"name": "actions-runner-linux-x64-<RUNNER_VERSION>.tar.gz",
"platform": "linux-x64",
"version": "<RUNNER_VERSION>",
"downloadUrl": "https://githubassets.azureedge.net/runners/<RUNNER_VERSION>/actions-runner-linux-x64-<RUNNER_VERSION>.tar.gz"
}
]

177
azure-pipelines-release.yml Normal file
View File

@@ -0,0 +1,177 @@
stages:
- stage: Build
jobs:
################################################################################
- job: build_windows_agent_x64
################################################################################
displayName: Windows Agent (x64)
pool:
vmImage: vs2017-win2016
steps:
# Steps template for windows platform
- template: windows.template.yml
# Package dotnet core windows dependency (VC++ Redistributable)
- powershell: |
Write-Host "Downloading 'VC++ Redistributable' package."
$outDir = Join-Path -Path $env:TMP -ChildPath ([Guid]::NewGuid())
New-Item -Path $outDir -ItemType directory
$outFile = Join-Path -Path $outDir -ChildPath "ucrt.zip"
Invoke-WebRequest -Uri https://vstsagenttools.blob.core.windows.net/tools/ucrt/ucrt_x64.zip -OutFile $outFile
Write-Host "Unzipping 'VC++ Redistributable' package to agent layout."
$unzipDir = Join-Path -Path $outDir -ChildPath "unzip"
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory($outFile, $unzipDir)
$agentLayoutBin = Join-Path -Path $(Build.SourcesDirectory) -ChildPath "_layout\bin"
Copy-Item -Path $unzipDir -Destination $agentLayoutBin -Force
displayName: Package UCRT
# Create agent package zip
- script: dev.cmd package Release
workingDirectory: src
displayName: Package Release
# Upload agent package zip as build artifact
- task: PublishBuildArtifacts@1
displayName: Publish Artifact (Windows)
inputs:
pathToPublish: _package
artifactName: runners
artifactType: container
################################################################################
- job: build_linux_agent_x64
################################################################################
displayName: Linux Agent (x64)
pool:
vmImage: ubuntu-16.04
steps:
# Steps template for non-windows platform
- template: nonwindows.template.yml
# Create agent package zip
- script: ./dev.sh package Release
workingDirectory: src
displayName: Package Release
# Upload agent package zip as build artifact
- task: PublishBuildArtifacts@1
displayName: Publish Artifact (Linux)
inputs:
pathToPublish: _package
artifactName: runners
artifactType: container
################################################################################
- job: build_osx_agent
################################################################################
displayName: macOS Agent (x64)
pool:
vmImage: macOS-10.13
steps:
# Steps template for non-windows platform
- template: nonwindows.template.yml
# Create agent package zip
- script: ./dev.sh package Release
workingDirectory: src
displayName: Package Release
# Upload agent package zip as build artifact
- task: PublishBuildArtifacts@1
displayName: Publish Artifact (OSX)
inputs:
pathToPublish: _package
artifactName: runners
artifactType: container
- stage: Release
dependsOn: Build
jobs:
################################################################################
- job: publish_agent_packages
################################################################################
displayName: Publish Agents (Windows/Linux/OSX)
pool:
name: ProductionRMAgents
steps:
# Download all agent packages from all previous phases
- task: DownloadBuildArtifacts@0
displayName: Download Agent Packages
inputs:
artifactName: runners
# Upload agent packages to Azure blob storage and refresh Azure CDN
- powershell: |
Write-Host "Preloading Azure modules." # This is for better performance, to avoid module-autoloading.
Import-Module AzureRM, AzureRM.profile, AzureRM.Storage, Azure.Storage, AzureRM.Cdn -ErrorAction Ignore -PassThru
Enable-AzureRmAlias -Scope CurrentUser
$uploadFiles = New-Object System.Collections.ArrayList
$certificateThumbprint = (Get-ItemProperty -Path "$(ServicePrincipalReg)").ServicePrincipalCertThumbprint
$clientId = (Get-ItemProperty -Path "$(ServicePrincipalReg)").ServicePrincipalClientId
Write-Host "##vso[task.setsecret]$certificateThumbprint"
Write-Host "##vso[task.setsecret]$clientId"
Login-AzureRmAccount -ServicePrincipal -CertificateThumbprint $certificateThumbprint -ApplicationId $clientId -TenantId $(GitHubTenantId)
Select-AzureRmSubscription -SubscriptionId $(GitHubSubscriptionId)
$storage = Get-AzureRmStorageAccount -ResourceGroupName githubassets -AccountName githubassets
Get-ChildItem -LiteralPath "$(System.ArtifactsDirectory)/runners" | ForEach-Object {
$versionDir = $_.Name.Trim('.zip').Trim('.tar.gz')
$versionDir = $versionDir.SubString($versionDir.LastIndexOf('-') + 1)
Write-Host "##vso[task.setvariable variable=ReleaseAgentVersion;]$versionDir"
Write-Host "Uploading $_ to BlobStorage githubassets/runners/$versionDir"
Set-AzureStorageBlobContent -Context $storage.Context -Container runners -File "$(System.ArtifactsDirectory)/runners/$_" -Blob "$versionDir/$_" -Force
$uploadFiles.Add("/runners/$versionDir/$_")
}
Write-Host "Get CDN info"
Get-AzureRmCdnEndpoint -ProfileName githubassets -ResourceGroupName githubassets
Write-Host "Purge Azure CDN Cache"
Unpublish-AzureRmCdnEndpointContent -EndpointName githubassets -ProfileName githubassets -ResourceGroupName githubassets -PurgeContent $uploadFiles
Write-Host "Pull assets through Azure CDN"
$uploadFiles | ForEach-Object {
$downloadUrl = "https://githubassets.azureedge.net" + $_
Write-Host $downloadUrl
Invoke-WebRequest -Uri $downloadUrl -OutFile $_.SubString($_.LastIndexOf('/') + 1)
}
displayName: Upload to Azure Blob
# Create agent release on Github
- powershell: |
Write-Host "Creating github release."
$releaseNotes = [System.IO.File]::ReadAllText("$(Build.SourcesDirectory)\releaseNote.md").Replace("<RUNNER_VERSION>","$(ReleaseAgentVersion)")
$releaseData = @{
tag_name = "v$(ReleaseAgentVersion)";
target_commitish = "$(Build.SourceVersion)";
name = "v$(ReleaseAgentVersion)";
body = $releaseNotes;
draft = $false;
prerelease = $true;
}
$releaseParams = @{
Uri = "https://api.github.com/repos/actions/runner/releases";
Method = 'POST';
Headers = @{
Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("github:$(GithubToken)"));
}
ContentType = 'application/json';
Body = (ConvertTo-Json $releaseData -Compress)
}
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$releaseCreated = Invoke-RestMethod @releaseParams
Write-Host $releaseCreated
$releaseId = $releaseCreated.id
$assets = [System.IO.File]::ReadAllText("$(Build.SourcesDirectory)\assets.json").Replace("<RUNNER_VERSION>","$(ReleaseAgentVersion)")
$assetsParams = @{
Uri = "https://uploads.github.com/repos/actions/runner/releases/$releaseId/assets?name=assets.json"
Method = 'POST';
Headers = @{
Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("github:$(GithubToken)"));
}
ContentType = 'application/octet-stream';
Body = [system.Text.Encoding]::UTF8.GetBytes($assets)
}
Invoke-RestMethod @assetsParams
displayName: Create agent release on Github

95
azure-pipelines.yml Normal file
View File

@@ -0,0 +1,95 @@
jobs:
################################################################################
- job: build_windows_x64_agent
################################################################################
displayName: Windows Agent (x64)
pool:
vmImage: vs2017-win2016
steps:
# Steps template for windows platform
- template: windows.template.yml
# Package dotnet core windows dependency (VC++ Redistributable)
- powershell: |
Write-Host "Downloading 'VC++ Redistributable' package."
$outDir = Join-Path -Path $env:TMP -ChildPath ([Guid]::NewGuid())
New-Item -Path $outDir -ItemType directory
$outFile = Join-Path -Path $outDir -ChildPath "ucrt.zip"
Invoke-WebRequest -Uri https://vstsagenttools.blob.core.windows.net/tools/ucrt/ucrt_x64.zip -OutFile $outFile
Write-Host "Unzipping 'VC++ Redistributable' package to agent layout."
$unzipDir = Join-Path -Path $outDir -ChildPath "unzip"
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory($outFile, $unzipDir)
$agentLayoutBin = Join-Path -Path $(Build.SourcesDirectory) -ChildPath "_layout\bin"
Copy-Item -Path $unzipDir -Destination $agentLayoutBin -Force
displayName: Package UCRT
condition: and(succeeded(), ne(variables['build.reason'], 'PullRequest'))
# Create agent package zip
- script: dev.cmd package Release
workingDirectory: src
displayName: Package Release
condition: and(succeeded(), ne(variables['build.reason'], 'PullRequest'))
# Upload agent package zip as build artifact
- task: PublishBuildArtifacts@1
displayName: Publish Artifact (Windows x64)
condition: and(succeeded(), ne(variables['build.reason'], 'PullRequest'))
inputs:
pathToPublish: _package
artifactName: agent
artifactType: container
################################################################################
- job: build_linux_x64_agent
################################################################################
displayName: Linux Agent (x64)
pool:
vmImage: ubuntu-16.04
steps:
# Steps template for non-windows platform
- template: nonwindows.template.yml
# Create agent package zip
- script: ./dev.sh package Release
workingDirectory: src
displayName: Package Release
condition: and(succeeded(), ne(variables['build.reason'], 'PullRequest'))
# Upload agent package zip as build artifact
- task: PublishBuildArtifacts@1
displayName: Publish Artifact (Linux x64)
condition: and(succeeded(), ne(variables['build.reason'], 'PullRequest'))
inputs:
pathToPublish: _package
artifactName: agent
artifactType: container
################################################################################
- job: build_osx_agent
################################################################################
displayName: macOS Agent (x64)
pool:
vmImage: macOS-10.14
steps:
# Steps template for non-windows platform
- template: nonwindows.template.yml
# Create agent package zip
- script: ./dev.sh package Release
workingDirectory: src
displayName: Package Release
condition: and(succeeded(), ne(variables['build.reason'], 'PullRequest'))
# Upload agent package zip as build artifact
- task: PublishBuildArtifacts@1
displayName: Publish Artifact (OSX)
condition: and(succeeded(), ne(variables['build.reason'], 'PullRequest'))
inputs:
pathToPublish: _package
artifactName: agent
artifactType: container

41
docs/contribute.md Normal file
View File

@@ -0,0 +1,41 @@
# Contribute (Dev)
## Dev Dependencies
![Win](res/win_sm.png) Git for Windows [Install Here](https://git-scm.com/downloads) (needed for dev sh script)
## Build, Test, Layout
From src:
![Win](res/win_sm.png) `dev {command}`
![*nix](res/linux_sm.png) `./dev.sh {command}`
**Commands:**
`layout` (`l`): Run first time to create a full agent layout in {root}/_layout
`build` (`b`): build everything and update agent layout folder
`test` (`t`): build agent binaries and run unit tests
Normal dev flow:
```bash
git clone https://github.com/actions/runner
cd ./src
./dev.(sh/cmd) layout # the agent that build from source is in {root}/_layout
<make code changes>
./dev.(sh/cmd) build # {root}/_layout will get updated
./dev.(sh/cmd) test # run all unit tests before git commit/push
```
## Editors
[Using Visual Studio 2019](https://www.visualstudio.com/vs/)
[Using Visual Studio Code](https://code.visualstudio.com/)
## Styling
We use the dotnet foundation and CoreCLR style guidelines [located here](
https://github.com/dotnet/corefx/blob/master/Documentation/coding-guidelines/coding-style.md)

BIN
docs/res/apple_med.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 KiB

BIN
docs/res/apple_sm.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 KiB

BIN
docs/res/github-graph.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 158 KiB

BIN
docs/res/linux_med.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.1 KiB

BIN
docs/res/linux_sm.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 KiB

BIN
docs/res/redhat_med.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.2 KiB

BIN
docs/res/redhat_sm.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 KiB

BIN
docs/res/ubuntu_med.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 KiB

BIN
docs/res/ubuntu_sm.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 KiB

BIN
docs/res/win_med.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.5 KiB

BIN
docs/res/win_sm.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 KiB

40
docs/start/envlinux.md Normal file
View File

@@ -0,0 +1,40 @@
# ![Linux](../res/linux_med.png) Linux System Prerequisites
## Supported Distributions and Versions
x64
- Red Hat Enterprise Linux 6 (see note 1), 7
- CentOS 6 (see note 1), 7
- Oracle Linux 7
- Fedora 28, 27
- Debian 9, 8.7 or later versions
- Ubuntu 18.04, Ubuntu 16.04, Ubuntu 14.04
- Linux Mint 18, 17
- openSUSE 42.3 or later versions
- SUSE Enterprise Linux (SLES) 12 SP2 or later versions
ARM32 (see note 2)
- Debian 9 or later versions
- Ubuntu 18.04 or later versions
> Note 1: Red Hat Enterprise Linux 6 and CentOS 6 require installing the specialized "rhel.6-x64" agent package
> Note 2: ARM instruction set [ARMv7](https://en.wikipedia.org/wiki/List_of_ARM_microarchitectures) or above is required, you can get your device's information by executing `uname -a`
## Install .Net Core 2.x Linux Dependencies
The `./config.sh` will check .Net Core 2.x dependencies during agent configuration.
You might see something like this which indicate a dependency's missing.
```bash
./config.sh
libunwind.so.8 => not found
libunwind-x86_64.so.8 => not found
Dependencies is missing for Dotnet Core 2.1
Execute ./bin/installdependencies.sh to install any missing Dotnet Core 2.1 dependencies.
```
You can easily correct the problem by executing `./bin/installdependencies.sh`.
The `installdependencies.sh` script should install all required dependencies on all supported Linux versions
> Note: The `installdependencies.sh` script will try to use the default package management mechanism on your Linux flavor (ex. `yum`/`apt-get`/`apt`). You might need to deal with error coming from the package management mechanism related to your setup, like [#1353](https://github.com/Microsoft/vsts-agent/issues/1353)
## [More .Net Core Prerequisites Information](https://docs.microsoft.com/en-us/dotnet/core/linux-prerequisites?tabs=netcore2x)

10
docs/start/envosx.md Normal file
View File

@@ -0,0 +1,10 @@
# ![osx](../res/apple_med.png) macOS/OS X System Prerequisites
## Supported Versions
- macOS Sierra (10.12) and later versions
## [More .Net Core Prerequisites Information](https://docs.microsoft.com/en-us/dotnet/core/macos-prerequisites?tabs=netcore2x)

12
docs/start/envwin.md Normal file
View File

@@ -0,0 +1,12 @@
# ![win](../res/win_med.png) Windows System Prerequisites
## Supported Versions
- Windows 7 64-bit
- Windows 8.1 64-bit
- Windows 10 64-bit
- Windows Server 2008 R2 SP1 64-bit
- Windows Server 2012 R2 64-bit
- Windows Server 2016 64-bit
## [More .Net Core Prerequisites Information](https://docs.microsoft.com/en-us/dotnet/core/windows-prerequisites?tabs=netcore2x)

7
images/arm/Dockerfile Normal file
View File

@@ -0,0 +1,7 @@
FROM mcr.microsoft.com/dotnet/core/runtime-deps:2.1
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
curl \
git \
&& rm -rf /var/lib/apt/lists/*

150
images/centos6/Dockerfile Normal file
View File

@@ -0,0 +1,150 @@
FROM centos:6
# Install dependencies
RUN yum install -y \
centos-release-SCL \
epel-release \
wget \
unzip \
&& \
rpm --import http://linuxsoft.cern.ch/cern/slc6X/x86_64/RPM-GPG-KEY-cern && \
wget -O /etc/yum.repos.d/slc6-devtoolset.repo http://linuxsoft.cern.ch/cern/devtoolset/slc6-devtoolset.repo && \
yum install -y \
"perl(Time::HiRes)" \
autoconf \
cmake \
cmake3 \
devtoolset-2-toolchain \
doxygen \
expat-devel \
gcc \
gcc-c++ \
gdb \
gettext-devel \
krb5-devel \
libedit-devel \
libidn-devel \
libmetalink-devel \
libnghttp2-devel \
libssh2-devel \
libunwind-devel \
libuuid-devel \
lttng-ust-devel \
lzma \
ncurses-devel \
openssl-devel \
perl-devel \
python-argparse \
python27 \
readline-devel \
swig \
xz \
zlib-devel \
&& \
yum clean all
# Build and install clang and lldb 3.9.1
RUN wget ftp://sourceware.org/pub/binutils/snapshots/binutils-2.29.1.tar.xz && \
wget http://releases.llvm.org/3.9.1/cfe-3.9.1.src.tar.xz && \
wget http://releases.llvm.org/3.9.1/llvm-3.9.1.src.tar.xz && \
wget http://releases.llvm.org/3.9.1/lldb-3.9.1.src.tar.xz && \
wget http://releases.llvm.org/3.9.1/compiler-rt-3.9.1.src.tar.xz && \
\
tar -xf binutils-2.29.1.tar.xz && \
tar -xf llvm-3.9.1.src.tar.xz && \
mkdir llvm-3.9.1.src/tools/clang && \
mkdir llvm-3.9.1.src/tools/lldb && \
mkdir llvm-3.9.1.src/projects/compiler-rt && \
tar -xf cfe-3.9.1.src.tar.xz --strip 1 -C llvm-3.9.1.src/tools/clang && \
tar -xf lldb-3.9.1.src.tar.xz --strip 1 -C llvm-3.9.1.src/tools/lldb && \
tar -xf compiler-rt-3.9.1.src.tar.xz --strip 1 -C llvm-3.9.1.src/projects/compiler-rt && \
rm binutils-2.29.1.tar.xz && \
rm cfe-3.9.1.src.tar.xz && \
rm lldb-3.9.1.src.tar.xz && \
rm llvm-3.9.1.src.tar.xz && \
rm compiler-rt-3.9.1.src.tar.xz && \
\
mkdir llvmbuild && \
cd llvmbuild && \
scl enable python27 devtoolset-2 \
' \
cmake3 \
-DCMAKE_CXX_COMPILER=/opt/rh/devtoolset-2/root/usr/bin/g++ \
-DCMAKE_C_COMPILER=/opt/rh/devtoolset-2/root/usr/bin/gcc \
-DCMAKE_LINKER=/opt/rh/devtoolset-2/root/usr/bin/ld \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_LIBDIR_SUFFIX=64 \
-DLLVM_ENABLE_EH=1 \
-DLLVM_ENABLE_RTTI=1 \
-DLLVM_BINUTILS_INCDIR=../binutils-2.29.1/include \
../llvm-3.9.1.src \
&& \
make -j $(($(getconf _NPROCESSORS_ONLN)+1)) && \
make install \
' && \
cd .. && \
rm -r llvmbuild && \
rm -r llvm-3.9.1.src && \
rm -r binutils-2.29.1
# Build and install curl 7.45.0
RUN wget https://curl.haxx.se/download/curl-7.45.0.tar.lzma && \
tar -xf curl-7.45.0.tar.lzma && \
rm curl-7.45.0.tar.lzma && \
cd curl-7.45.0 && \
scl enable python27 devtoolset-2 \
' \
./configure \
--disable-dict \
--disable-ftp \
--disable-gopher \
--disable-imap \
--disable-ldap \
--disable-ldaps \
--disable-libcurl-option \
--disable-manual \
--disable-pop3 \
--disable-rtsp \
--disable-smb \
--disable-smtp \
--disable-telnet \
--disable-tftp \
--enable-ipv6 \
--enable-optimize \
--enable-symbol-hiding \
--with-ca-bundle=/etc/pki/tls/certs/ca-bundle.crt \
--with-nghttp2 \
--with-gssapi \
--with-ssl \
--without-librtmp \
&& \
make install \
' && \
cd .. && \
rm -r curl-7.45.0
# Install ICU 57.1
RUN wget http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-RHEL6-x64.tgz && \
tar -xf icu4c-57_1-RHEL6-x64.tgz -C / && \
rm icu4c-57_1-RHEL6-x64.tgz
# Compile and install a version of the git that supports the features that cli repo build needs
# NOTE: The git needs to be built after the curl so that it can use the libcurl to add https
# protocol support.
RUN \
wget https://www.kernel.org/pub/software/scm/git/git-2.9.5.tar.gz && \
tar -xf git-2.9.5.tar.gz && \
rm git-2.9.5.tar.gz && \
cd git-2.9.5 && \
make configure && \
./configure --prefix=/usr/local --without-tcltk && \
make -j $(nproc --all) all && \
make install && \
cd .. && \
rm -r git-2.9.5
ENV LD_LIBRARY_PATH=/usr/local/lib

29
nonwindows.template.yml Normal file
View File

@@ -0,0 +1,29 @@
steps:
# Build agent layout
- script: ./dev.sh layout Release
workingDirectory: src
displayName: Build & Layout Release
# Run test
- script: ./dev.sh test
workingDirectory: src
displayName: Test
# # Publish test results
# - task: PublishTestResults@2
# displayName: Publish Test Results **/*.trx
# condition: always()
# inputs:
# testRunner: VSTest
# testResultsFiles: '**/*.trx'
# testRunTitle: 'Agent Tests'
# # Upload test log
# - task: PublishBuildArtifacts@1
# displayName: Publish Test logs
# condition: always()
# inputs:
# pathToPublish: src/Test/TestLogs
# artifactName: $(System.JobId)
# artifactType: container

44
releaseNote.md Normal file
View File

@@ -0,0 +1,44 @@
## Features
- Runner config auth via GitHub.com. (#107) (#117)
- Adding wrapper action to support post job cleanup, adding checkout v1.1 (#91)
- Improving terminal experience (#110)
- Add runner support for cache action. (#120)
## Bugs
- Set GITHUB_ACTIONS in containers. (#119)
- Fix issue data column/col mismatch. (#122)
## Misc
- Use GitHub actions for CI/PR (#112)
- Code Cleanup (#123) (#124) (#125)
## Agent Downloads
| | Package |
| ------- | ----------------------------------------------------------------------------------------------------------- |
| Windows x64 | [actions-runner-win-x64-<RUNNER_VERSION>.zip](https://githubassets.azureedge.net/runners/<RUNNER_VERSION>/actions-runner-win-x64-<RUNNER_VERSION>.zip) |
| macOS | [actions-runner-osx-x64-<RUNNER_VERSION>.tar.gz](https://githubassets.azureedge.net/runners/<RUNNER_VERSION>/actions-runner-osx-x64-<RUNNER_VERSION>.tar.gz) |
| Linux x64 | [actions-runner-linux-x64-<RUNNER_VERSION>.tar.gz](https://githubassets.azureedge.net/runners/<RUNNER_VERSION>/actions-runner-linux-x64-<RUNNER_VERSION>.tar.gz) |
After Download:
## Windows x64
``` bash
C:\> mkdir myagent && cd myagent
C:\myagent> Add-Type -AssemblyName System.IO.Compression.FileSystem ; [System.IO.Compression.ZipFile]::ExtractToDirectory("$HOME\Downloads\actions-runner-win-x64-<RUNNER_VERSION>.zip", "$PWD")
```
## OSX
``` bash
~/$ mkdir myagent && cd myagent
~/myagent$ tar xzf ~/Downloads/actions-runner-osx-x64-<RUNNER_VERSION>.tar.gz
```
## Linux x64
``` bash
~/$ mkdir myagent && cd myagent
~/myagent$ tar xzf ~/Downloads/actions-runner-linux-x64-<RUNNER_VERSION>.tar.gz
```

650
src/Misc/dotnet-install.ps1 vendored Normal file
View File

@@ -0,0 +1,650 @@
#
# Copyright (c) .NET Foundation and contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
<#
.SYNOPSIS
Installs dotnet cli
.DESCRIPTION
Installs dotnet cli. If dotnet installation already exists in the given directory
it will update it only if the requested version differs from the one already installed.
.PARAMETER Channel
Default: LTS
Download from the Channel specified. Possible values:
- Current - most current release
- LTS - most current supported release
- 2-part version in a format A.B - represents a specific release
examples: 2.0, 1.0
- Branch name
examples: release/2.0.0, Master
Note: The version parameter overrides the channel parameter.
.PARAMETER Version
Default: latest
Represents a build version on specific channel. Possible values:
- latest - most latest build on specific channel
- coherent - most latest coherent build on specific channel
coherent applies only to SDK downloads
- 3-part version in a format A.B.C - represents specific version of build
examples: 2.0.0-preview2-006120, 1.1.0
.PARAMETER InstallDir
Default: %LocalAppData%\Microsoft\dotnet
Path to where to install dotnet. Note that binaries will be placed directly in a given directory.
.PARAMETER Architecture
Default: <auto> - this value represents currently running OS architecture
Architecture of dotnet binaries to be installed.
Possible values are: <auto>, amd64, x64, x86, arm64, arm
.PARAMETER SharedRuntime
This parameter is obsolete and may be removed in a future version of this script.
The recommended alternative is '-Runtime dotnet'.
Default: false
Installs just the shared runtime bits, not the entire SDK.
This is equivalent to specifying `-Runtime dotnet`.
.PARAMETER Runtime
Installs just a shared runtime, not the entire SDK.
Possible values:
- dotnet - the Microsoft.NETCore.App shared runtime
- aspnetcore - the Microsoft.AspNetCore.App shared runtime
- windowsdesktop - the Microsoft.WindowsDesktop.App shared runtime
.PARAMETER DryRun
If set it will not perform installation but instead display what command line to use to consistently install
currently requested version of dotnet cli. In example if you specify version 'latest' it will display a link
with specific version so that this command can be used deterministicly in a build script.
It also displays binaries location if you prefer to install or download it yourself.
.PARAMETER NoPath
By default this script will set environment variable PATH for the current process to the binaries folder inside installation folder.
If set it will display binaries location but not set any environment variable.
.PARAMETER Verbose
Displays diagnostics information.
.PARAMETER AzureFeed
Default: https://dotnetcli.azureedge.net/dotnet
This parameter typically is not changed by the user.
It allows changing the URL for the Azure feed used by this installer.
.PARAMETER UncachedFeed
This parameter typically is not changed by the user.
It allows changing the URL for the Uncached feed used by this installer.
.PARAMETER FeedCredential
Used as a query string to append to the Azure feed.
It allows changing the URL to use non-public blob storage accounts.
.PARAMETER ProxyAddress
If set, the installer will use the proxy when making web requests
.PARAMETER ProxyUseDefaultCredentials
Default: false
Use default credentials, when using proxy address.
.PARAMETER SkipNonVersionedFiles
Default: false
Skips installing non-versioned files if they already exist, such as dotnet.exe.
.PARAMETER NoCdn
Disable downloading from the Azure CDN, and use the uncached feed directly.
#>
[cmdletbinding()]
param(
[string]$Channel="LTS",
[string]$Version="Latest",
[string]$InstallDir="<auto>",
[string]$Architecture="<auto>",
[ValidateSet("dotnet", "aspnetcore", "windowsdesktop", IgnoreCase = $false)]
[string]$Runtime,
[Obsolete("This parameter may be removed in a future version of this script. The recommended alternative is '-Runtime dotnet'.")]
[switch]$SharedRuntime,
[switch]$DryRun,
[switch]$NoPath,
[string]$AzureFeed="https://dotnetcli.azureedge.net/dotnet",
[string]$UncachedFeed="https://dotnetcli.blob.core.windows.net/dotnet",
[string]$FeedCredential,
[string]$ProxyAddress,
[switch]$ProxyUseDefaultCredentials,
[switch]$SkipNonVersionedFiles,
[switch]$NoCdn
)
Set-StrictMode -Version Latest
$ErrorActionPreference="Stop"
$ProgressPreference="SilentlyContinue"
if ($NoCdn) {
$AzureFeed = $UncachedFeed
}
$BinFolderRelativePath=""
if ($SharedRuntime -and (-not $Runtime)) {
$Runtime = "dotnet"
}
# example path with regex: shared/1.0.0-beta-12345/somepath
$VersionRegEx="/\d+\.\d+[^/]+/"
$OverrideNonVersionedFiles = !$SkipNonVersionedFiles
function Say($str) {
Write-Host "dotnet-install: $str"
}
function Say-Verbose($str) {
Write-Verbose "dotnet-install: $str"
}
function Say-Invocation($Invocation) {
$command = $Invocation.MyCommand;
$args = (($Invocation.BoundParameters.Keys | foreach { "-$_ `"$($Invocation.BoundParameters[$_])`"" }) -join " ")
Say-Verbose "$command $args"
}
function Invoke-With-Retry([ScriptBlock]$ScriptBlock, [int]$MaxAttempts = 3, [int]$SecondsBetweenAttempts = 1) {
$Attempts = 0
while ($true) {
try {
return $ScriptBlock.Invoke()
}
catch {
$Attempts++
if ($Attempts -lt $MaxAttempts) {
Start-Sleep $SecondsBetweenAttempts
}
else {
throw
}
}
}
}
function Get-Machine-Architecture() {
Say-Invocation $MyInvocation
# possible values: amd64, x64, x86, arm64, arm
return $ENV:PROCESSOR_ARCHITECTURE
}
function Get-CLIArchitecture-From-Architecture([string]$Architecture) {
Say-Invocation $MyInvocation
switch ($Architecture.ToLower()) {
{ $_ -eq "<auto>" } { return Get-CLIArchitecture-From-Architecture $(Get-Machine-Architecture) }
{ ($_ -eq "amd64") -or ($_ -eq "x64") } { return "x64" }
{ $_ -eq "x86" } { return "x86" }
{ $_ -eq "arm" } { return "arm" }
{ $_ -eq "arm64" } { return "arm64" }
default { throw "Architecture not supported. If you think this is a bug, report it at https://github.com/dotnet/cli/issues" }
}
}
# The version text returned from the feeds is a 1-line or 2-line string:
# For the SDK and the dotnet runtime (2 lines):
# Line 1: # commit_hash
# Line 2: # 4-part version
# For the aspnetcore runtime (1 line):
# Line 1: # 4-part version
function Get-Version-Info-From-Version-Text([string]$VersionText) {
Say-Invocation $MyInvocation
$Data = -split $VersionText
$VersionInfo = @{
CommitHash = $(if ($Data.Count -gt 1) { $Data[0] })
Version = $Data[-1] # last line is always the version number.
}
return $VersionInfo
}
function Load-Assembly([string] $Assembly) {
try {
Add-Type -Assembly $Assembly | Out-Null
}
catch {
# On Nano Server, Powershell Core Edition is used. Add-Type is unable to resolve base class assemblies because they are not GAC'd.
# Loading the base class assemblies is not unnecessary as the types will automatically get resolved.
}
}
function GetHTTPResponse([Uri] $Uri)
{
Invoke-With-Retry(
{
$HttpClient = $null
try {
# HttpClient is used vs Invoke-WebRequest in order to support Nano Server which doesn't support the Invoke-WebRequest cmdlet.
Load-Assembly -Assembly System.Net.Http
if(-not $ProxyAddress) {
try {
# Despite no proxy being explicitly specified, we may still be behind a default proxy
$DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy;
if($DefaultProxy -and (-not $DefaultProxy.IsBypassed($Uri))) {
$ProxyAddress = $DefaultProxy.GetProxy($Uri).OriginalString
$ProxyUseDefaultCredentials = $true
}
} catch {
# Eat the exception and move forward as the above code is an attempt
# at resolving the DefaultProxy that may not have been a problem.
$ProxyAddress = $null
Say-Verbose("Exception ignored: $_.Exception.Message - moving forward...")
}
}
if($ProxyAddress) {
$HttpClientHandler = New-Object System.Net.Http.HttpClientHandler
$HttpClientHandler.Proxy = New-Object System.Net.WebProxy -Property @{Address=$ProxyAddress;UseDefaultCredentials=$ProxyUseDefaultCredentials}
$HttpClient = New-Object System.Net.Http.HttpClient -ArgumentList $HttpClientHandler
}
else {
$HttpClient = New-Object System.Net.Http.HttpClient
}
# Default timeout for HttpClient is 100s. For a 50 MB download this assumes 500 KB/s average, any less will time out
# 20 minutes allows it to work over much slower connections.
$HttpClient.Timeout = New-TimeSpan -Minutes 20
$Response = $HttpClient.GetAsync("${Uri}${FeedCredential}").Result
if (($Response -eq $null) -or (-not ($Response.IsSuccessStatusCode))) {
# The feed credential is potentially sensitive info. Do not log FeedCredential to console output.
$ErrorMsg = "Failed to download $Uri."
if ($Response -ne $null) {
$ErrorMsg += " $Response"
}
throw $ErrorMsg
}
return $Response
}
finally {
if ($HttpClient -ne $null) {
$HttpClient.Dispose()
}
}
})
}
function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel, [bool]$Coherent) {
Say-Invocation $MyInvocation
$VersionFileUrl = $null
if ($Runtime -eq "dotnet") {
$VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version"
}
elseif ($Runtime -eq "aspnetcore") {
$VersionFileUrl = "$UncachedFeed/aspnetcore/Runtime/$Channel/latest.version"
}
# Currently, the WindowsDesktop runtime is manufactured with the .Net core runtime
elseif ($Runtime -eq "windowsdesktop") {
$VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version"
}
elseif (-not $Runtime) {
if ($Coherent) {
$VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.coherent.version"
}
else {
$VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.version"
}
}
else {
throw "Invalid value for `$Runtime"
}
try {
$Response = GetHTTPResponse -Uri $VersionFileUrl
}
catch {
throw "Could not resolve version information."
}
$StringContent = $Response.Content.ReadAsStringAsync().Result
switch ($Response.Content.Headers.ContentType) {
{ ($_ -eq "application/octet-stream") } { $VersionText = $StringContent }
{ ($_ -eq "text/plain") } { $VersionText = $StringContent }
{ ($_ -eq "text/plain; charset=UTF-8") } { $VersionText = $StringContent }
default { throw "``$Response.Content.Headers.ContentType`` is an unknown .version file content type." }
}
$VersionInfo = Get-Version-Info-From-Version-Text $VersionText
return $VersionInfo
}
function Get-Specific-Version-From-Version([string]$AzureFeed, [string]$Channel, [string]$Version) {
Say-Invocation $MyInvocation
switch ($Version.ToLower()) {
{ $_ -eq "latest" } {
$LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $False
return $LatestVersionInfo.Version
}
{ $_ -eq "coherent" } {
$LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $True
return $LatestVersionInfo.Version
}
default { return $Version }
}
}
function Get-Download-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) {
Say-Invocation $MyInvocation
if ($Runtime -eq "dotnet") {
$PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-runtime-$SpecificVersion-win-$CLIArchitecture.zip"
}
elseif ($Runtime -eq "aspnetcore") {
$PayloadURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/aspnetcore-runtime-$SpecificVersion-win-$CLIArchitecture.zip"
}
elseif ($Runtime -eq "windowsdesktop") {
$PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/windowsdesktop-runtime-$SpecificVersion-win-$CLIArchitecture.zip"
}
elseif (-not $Runtime) {
$PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-sdk-$SpecificVersion-win-$CLIArchitecture.zip"
}
else {
throw "Invalid value for `$Runtime"
}
Say-Verbose "Constructed primary named payload URL: $PayloadURL"
return $PayloadURL
}
function Get-LegacyDownload-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) {
Say-Invocation $MyInvocation
if (-not $Runtime) {
$PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-dev-win-$CLIArchitecture.$SpecificVersion.zip"
}
elseif ($Runtime -eq "dotnet") {
$PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-win-$CLIArchitecture.$SpecificVersion.zip"
}
else {
return $null
}
Say-Verbose "Constructed legacy named payload URL: $PayloadURL"
return $PayloadURL
}
function Get-User-Share-Path() {
Say-Invocation $MyInvocation
$InstallRoot = $env:DOTNET_INSTALL_DIR
if (!$InstallRoot) {
$InstallRoot = "$env:LocalAppData\Microsoft\dotnet"
}
return $InstallRoot
}
function Resolve-Installation-Path([string]$InstallDir) {
Say-Invocation $MyInvocation
if ($InstallDir -eq "<auto>") {
return Get-User-Share-Path
}
return $InstallDir
}
function Get-Version-Info-From-Version-File([string]$InstallRoot, [string]$RelativePathToVersionFile) {
Say-Invocation $MyInvocation
$VersionFile = Join-Path -Path $InstallRoot -ChildPath $RelativePathToVersionFile
Say-Verbose "Local version file: $VersionFile"
if (Test-Path $VersionFile) {
$VersionText = cat $VersionFile
Say-Verbose "Local version file text: $VersionText"
return Get-Version-Info-From-Version-Text $VersionText
}
Say-Verbose "Local version file not found."
return $null
}
function Is-Dotnet-Package-Installed([string]$InstallRoot, [string]$RelativePathToPackage, [string]$SpecificVersion) {
Say-Invocation $MyInvocation
$DotnetPackagePath = Join-Path -Path $InstallRoot -ChildPath $RelativePathToPackage | Join-Path -ChildPath $SpecificVersion
Say-Verbose "Is-Dotnet-Package-Installed: Path to a package: $DotnetPackagePath"
return Test-Path $DotnetPackagePath -PathType Container
}
function Get-Absolute-Path([string]$RelativeOrAbsolutePath) {
# Too much spam
# Say-Invocation $MyInvocation
return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($RelativeOrAbsolutePath)
}
function Get-Path-Prefix-With-Version($path) {
$match = [regex]::match($path, $VersionRegEx)
if ($match.Success) {
return $entry.FullName.Substring(0, $match.Index + $match.Length)
}
return $null
}
function Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package([System.IO.Compression.ZipArchive]$Zip, [string]$OutPath) {
Say-Invocation $MyInvocation
$ret = @()
foreach ($entry in $Zip.Entries) {
$dir = Get-Path-Prefix-With-Version $entry.FullName
if ($dir -ne $null) {
$path = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $dir)
if (-Not (Test-Path $path -PathType Container)) {
$ret += $dir
}
}
}
$ret = $ret | Sort-Object | Get-Unique
$values = ($ret | foreach { "$_" }) -join ";"
Say-Verbose "Directories to unpack: $values"
return $ret
}
# Example zip content and extraction algorithm:
# Rule: files if extracted are always being extracted to the same relative path locally
# .\
# a.exe # file does not exist locally, extract
# b.dll # file exists locally, override only if $OverrideFiles set
# aaa\ # same rules as for files
# ...
# abc\1.0.0\ # directory contains version and exists locally
# ... # do not extract content under versioned part
# abc\asd\ # same rules as for files
# ...
# def\ghi\1.0.1\ # directory contains version and does not exist locally
# ... # extract content
function Extract-Dotnet-Package([string]$ZipPath, [string]$OutPath) {
Say-Invocation $MyInvocation
Load-Assembly -Assembly System.IO.Compression.FileSystem
Set-Variable -Name Zip
try {
$Zip = [System.IO.Compression.ZipFile]::OpenRead($ZipPath)
$DirectoriesToUnpack = Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package -Zip $Zip -OutPath $OutPath
foreach ($entry in $Zip.Entries) {
$PathWithVersion = Get-Path-Prefix-With-Version $entry.FullName
if (($PathWithVersion -eq $null) -Or ($DirectoriesToUnpack -contains $PathWithVersion)) {
$DestinationPath = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $entry.FullName)
$DestinationDir = Split-Path -Parent $DestinationPath
$OverrideFiles=$OverrideNonVersionedFiles -Or (-Not (Test-Path $DestinationPath))
if ((-Not $DestinationPath.EndsWith("\")) -And $OverrideFiles) {
New-Item -ItemType Directory -Force -Path $DestinationDir | Out-Null
[System.IO.Compression.ZipFileExtensions]::ExtractToFile($entry, $DestinationPath, $OverrideNonVersionedFiles)
}
}
}
}
finally {
if ($Zip -ne $null) {
$Zip.Dispose()
}
}
}
function DownloadFile($Source, [string]$OutPath) {
if ($Source -notlike "http*") {
# Using System.IO.Path.GetFullPath to get the current directory
# does not work in this context - $pwd gives the current directory
if (![System.IO.Path]::IsPathRooted($Source)) {
$Source = $(Join-Path -Path $pwd -ChildPath $Source)
}
$Source = Get-Absolute-Path $Source
Say "Copying file from $Source to $OutPath"
Copy-Item $Source $OutPath
return
}
$Stream = $null
try {
$Response = GetHTTPResponse -Uri $Source
$Stream = $Response.Content.ReadAsStreamAsync().Result
$File = [System.IO.File]::Create($OutPath)
$Stream.CopyTo($File)
$File.Close()
}
finally {
if ($Stream -ne $null) {
$Stream.Dispose()
}
}
}
function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot, [string]$BinFolderRelativePath) {
$BinPath = Get-Absolute-Path $(Join-Path -Path $InstallRoot -ChildPath $BinFolderRelativePath)
if (-Not $NoPath) {
$SuffixedBinPath = "$BinPath;"
if (-Not $env:path.Contains($SuffixedBinPath)) {
Say "Adding to current process PATH: `"$BinPath`". Note: This change will not be visible if PowerShell was run as a child process."
$env:path = $SuffixedBinPath + $env:path
} else {
Say-Verbose "Current process PATH already contains `"$BinPath`""
}
}
else {
Say "Binaries of dotnet can be found in $BinPath"
}
}
$CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture
$SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $AzureFeed -Channel $Channel -Version $Version
$DownloadLink = Get-Download-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture
$LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture
$InstallRoot = Resolve-Installation-Path $InstallDir
Say-Verbose "InstallRoot: $InstallRoot"
$ScriptName = $MyInvocation.MyCommand.Name
if ($DryRun) {
Say "Payload URLs:"
Say "Primary named payload URL: $DownloadLink"
if ($LegacyDownloadLink) {
Say "Legacy named payload URL: $LegacyDownloadLink"
}
$RepeatableCommand = ".\$ScriptName -Version `"$SpecificVersion`" -InstallDir `"$InstallRoot`" -Architecture `"$CLIArchitecture`""
if ($Runtime -eq "dotnet") {
$RepeatableCommand+=" -Runtime `"dotnet`""
}
elseif ($Runtime -eq "aspnetcore") {
$RepeatableCommand+=" -Runtime `"aspnetcore`""
}
foreach ($key in $MyInvocation.BoundParameters.Keys) {
if (-not (@("Architecture","Channel","DryRun","InstallDir","Runtime","SharedRuntime","Version") -contains $key)) {
$RepeatableCommand+=" -$key `"$($MyInvocation.BoundParameters[$key])`""
}
}
Say "Repeatable invocation: $RepeatableCommand"
exit 0
}
if ($Runtime -eq "dotnet") {
$assetName = ".NET Core Runtime"
$dotnetPackageRelativePath = "shared\Microsoft.NETCore.App"
}
elseif ($Runtime -eq "aspnetcore") {
$assetName = "ASP.NET Core Runtime"
$dotnetPackageRelativePath = "shared\Microsoft.AspNetCore.App"
}
elseif ($Runtime -eq "windowsdesktop") {
$assetName = ".NET Core Windows Desktop Runtime"
$dotnetPackageRelativePath = "shared\Microsoft.WindowsDesktop.App"
}
elseif (-not $Runtime) {
$assetName = ".NET Core SDK"
$dotnetPackageRelativePath = "sdk"
}
else {
throw "Invalid value for `$Runtime"
}
# Check if the SDK version is already installed.
$isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion
if ($isAssetInstalled) {
Say "$assetName version $SpecificVersion is already installed."
Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath
exit 0
}
New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null
$installDrive = $((Get-Item $InstallRoot).PSDrive.Name);
$diskInfo = Get-PSDrive -Name $installDrive
if ($diskInfo.Free / 1MB -le 100) {
Say "There is not enough disk space on drive ${installDrive}:"
exit 0
}
$ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName())
Say-Verbose "Zip path: $ZipPath"
$DownloadFailed = $false
Say "Downloading link: $DownloadLink"
try {
DownloadFile -Source $DownloadLink -OutPath $ZipPath
}
catch {
Say "Cannot download: $DownloadLink"
if ($LegacyDownloadLink) {
$DownloadLink = $LegacyDownloadLink
$ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName())
Say-Verbose "Legacy zip path: $ZipPath"
Say "Downloading legacy link: $DownloadLink"
try {
DownloadFile -Source $DownloadLink -OutPath $ZipPath
}
catch {
Say "Cannot download: $DownloadLink"
$DownloadFailed = $true
}
}
else {
$DownloadFailed = $true
}
}
if ($DownloadFailed) {
throw "Could not find/download: `"$assetName`" with version = $SpecificVersion`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support"
}
Say "Extracting zip from $DownloadLink"
Extract-Dotnet-Package -ZipPath $ZipPath -OutPath $InstallRoot
# Check if the SDK version is now installed; if not, fail the installation.
$isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion
if (!$isAssetInstalled) {
throw "`"$assetName`" with version = $SpecificVersion failed to install with an unknown error."
}
Remove-Item $ZipPath
Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath
Say "Installation finished"
exit 0

1025
src/Misc/dotnet-install.sh vendored Executable file

File diff suppressed because it is too large Load Diff

148
src/Misc/externals.sh Executable file
View File

@@ -0,0 +1,148 @@
#!/bin/bash
PACKAGERUNTIME=$1
PRECACHE=$2
NODE_URL=https://nodejs.org/dist
NODE12_VERSION="12.4.0"
get_abs_path() {
# exploits the fact that pwd will print abs path when no args
echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
}
LAYOUT_DIR=$(get_abs_path "$(dirname $0)/../../_layout")
DOWNLOAD_DIR="$(get_abs_path "$(dirname $0)/../../_downloads")/netcore2x"
function failed() {
local error=${1:-Undefined error}
echo "Failed: $error" >&2
exit 1
}
function checkRC() {
local rc=$?
if [ $rc -ne 0 ]; then
failed "${1} failed with return code $rc"
fi
}
function acquireExternalTool() {
local download_source=$1 # E.g. https://github.com/microsoft/vswhere/releases/download/2.6.7/vswhere.exe
local target_dir="$LAYOUT_DIR/externals/$2" # E.g. $LAYOUT_DIR/externals/vswhere
local fix_nested_dir=$3 # Flag that indicates whether to move nested contents up one directory.
# Extract the portion of the URL after the protocol. E.g. github.com/microsoft/vswhere/releases/download/2.6.7/vswhere.exe
local relative_url="${download_source#*://}"
# Check if the download already exists.
local download_target="$DOWNLOAD_DIR/$relative_url"
local download_basename="$(basename "$download_target")"
local download_dir="$(dirname "$download_target")"
if [[ "$PRECACHE" != "" ]]; then
if [ -f "$download_target" ]; then
echo "Download exists: $download_basename"
else
# Delete any previous partial file.
local partial_target="$DOWNLOAD_DIR/partial/$download_basename"
mkdir -p "$(dirname "$partial_target")" || checkRC 'mkdir'
if [ -f "$partial_target" ]; then
rm "$partial_target" || checkRC 'rm'
fi
# Download from source to the partial file.
echo "Downloading $download_source"
mkdir -p "$(dirname "$download_target")" || checkRC 'mkdir'
# curl -f Fail silently (no output at all) on HTTP errors (H)
# -k Allow connections to SSL sites without certs (H)
# -S Show error. With -s, make curl show errors when they occur
# -L Follow redirects (H)
# -o FILE Write to FILE instead of stdout
curl -fkSL -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl'
# Move the partial file to the download target.
mv "$partial_target" "$download_target" || checkRC 'mv'
# Extract to current directory
# Ensure we can extract those files
# We might use them during dev.sh
if [[ "$download_basename" == *.zip ]]; then
# Extract the zip.
echo "Testing zip"
unzip "$download_target" -d "$download_dir" > /dev/null
local rc=$?
if [[ $rc -ne 0 && $rc -ne 1 ]]; then
failed "unzip failed with return code $rc"
fi
elif [[ "$download_basename" == *.tar.gz ]]; then
# Extract the tar gz.
echo "Testing tar gz"
tar xzf "$download_target" -C "$download_dir" > /dev/null || checkRC 'tar'
fi
fi
else
# Extract to layout.
mkdir -p "$target_dir" || checkRC 'mkdir'
local nested_dir=""
if [[ "$download_basename" == *.zip ]]; then
# Extract the zip.
echo "Extracting zip to layout"
unzip "$download_target" -d "$target_dir" > /dev/null
local rc=$?
if [[ $rc -ne 0 && $rc -ne 1 ]]; then
failed "unzip failed with return code $rc"
fi
# Capture the nested directory path if the fix_nested_dir flag is set.
if [[ "$fix_nested_dir" == "fix_nested_dir" ]]; then
nested_dir="${download_basename%.zip}" # Remove the trailing ".zip".
fi
elif [[ "$download_basename" == *.tar.gz ]]; then
# Extract the tar gz.
echo "Extracting tar gz to layout"
tar xzf "$download_target" -C "$target_dir" > /dev/null || checkRC 'tar'
# Capture the nested directory path if the fix_nested_dir flag is set.
if [[ "$fix_nested_dir" == "fix_nested_dir" ]]; then
nested_dir="${download_basename%.tar.gz}" # Remove the trailing ".tar.gz".
fi
else
# Copy the file.
echo "Copying to layout"
cp "$download_target" "$target_dir/" || checkRC 'cp'
fi
# Fixup the nested directory.
if [[ "$nested_dir" != "" ]]; then
if [ -d "$target_dir/$nested_dir" ]; then
mv "$target_dir/$nested_dir"/* "$target_dir/" || checkRC 'mv'
rmdir "$target_dir/$nested_dir" || checkRC 'rmdir'
fi
fi
fi
}
# Download the external tools only for Windows.
if [[ "$PACKAGERUNTIME" == "win-x64" ]]; then
acquireExternalTool "$NODE_URL/v${NODE12_VERSION}/win-x64/node.exe" node12/bin
acquireExternalTool "$NODE_URL/v${NODE12_VERSION}/win-x64/node.lib" node12/bin
if [[ "$PRECACHE" != "" ]]; then
acquireExternalTool "https://github.com/microsoft/vswhere/releases/download/2.6.7/vswhere.exe" vswhere
fi
fi
# Download the external tools only for OSX.
if [[ "$PACKAGERUNTIME" == "osx-x64" ]]; then
acquireExternalTool "$NODE_URL/v${NODE12_VERSION}/node-v${NODE12_VERSION}-darwin-x64.tar.gz" node12 fix_nested_dir
fi
# Download the external tools common across Linux PACKAGERUNTIMEs (excluding OSX).
if [[ "$PACKAGERUNTIME" == "linux-x64" || "$PACKAGERUNTIME" == "rhel.6-x64" ]]; then
acquireExternalTool "$NODE_URL/v${NODE12_VERSION}/node-v${NODE12_VERSION}-linux-x64.tar.gz" node12 fix_nested_dir
# TODO: Repath this blob to use a consistent version format (_ vs .)
acquireExternalTool "https://vstsagenttools.blob.core.windows.net/tools/nodejs/12_4_0/alpine/node-v${NODE12_VERSION}-alpine.tar.gz" node12_alpine
fi
if [[ "$PACKAGERUNTIME" == "linux-arm" ]]; then
acquireExternalTool "$NODE_URL/v${NODE12_VERSION}/node-v${NODE12_VERSION}-linux-armv7l.tar.gz" node12 fix_nested_dir
fi

View File

@@ -0,0 +1,91 @@
#!/usr/bin/env node
// Copyright (c) GitHub. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
var childProcess = require("child_process");
var path = require("path")
var supported = ['linux', 'darwin']
if (supported.indexOf(process.platform) == -1) {
console.log('Unsupported platform: ' + process.platform);
console.log('Supported platforms are: ' + supported.toString());
process.exit(1);
}
var stopping = false;
var listener = null;
var runService = function() {
var listenerExePath = path.join(__dirname, '../bin/Runner.Listener');
var interactive = process.argv[2] === "interactive";
if(!stopping) {
try {
if (interactive) {
console.log('Starting Runner listener interactively');
listener = childProcess.spawn(listenerExePath, ['run'], { env: process.env });
} else {
console.log('Starting Runner listener with startup type: service');
listener = childProcess.spawn(listenerExePath, ['run', '--startuptype', 'service'], { env: process.env });
}
console.log('Started listener process');
listener.stdout.on('data', (data) => {
process.stdout.write(data.toString('utf8'));
});
listener.stderr.on('data', (data) => {
process.stdout.write(data.toString('utf8'));
});
listener.on('close', (code) => {
console.log(`Runner listener exited with error code ${code}`);
if (code === 0) {
console.log('Runner listener exit with 0 return code, stop the service, no retry needed.');
stopping = true;
} else if (code === 1) {
console.log('Runner listener exit with terminated error, stop the service, no retry needed.');
stopping = true;
} else if (code === 2) {
console.log('Runner listener exit with retryable error, re-launch runner in 5 seconds.');
} else if (code === 3) {
console.log('Runner listener exit because of updating, re-launch runner in 5 seconds.');
} else {
console.log('Runner listener exit with undefined return code, re-launch runner in 5 seconds.');
}
if(!stopping) {
setTimeout(runService, 5000);
}
});
} catch(ex) {
console.log(ex);
}
}
}
runService();
console.log('Started running service');
var gracefulShutdown = function(code) {
console.log('Shutting down runner listener');
stopping = true;
if (listener) {
console.log('Sending SIGINT to runner listener to stop');
listener.kill('SIGINT');
// TODO wait for 30 seconds and send a SIGKILL
}
}
process.on('SIGINT', () => {
gracefulShutdown(0);
});
process.on('SIGTERM', () => {
gracefulShutdown(0);
});

View File

@@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>{{SvcName}}</string>
<key>ProgramArguments</key>
<array>
<string>{{RunnerRoot}}/runsvc.sh</string>
</array>
<key>UserName</key>
<string>{{User}}</string>
<key>WorkingDirectory</key>
<string>{{RunnerRoot}}</string>
<key>RunAtLoad</key>
<true/>
<key>StandardOutPath</key>
<string>{{UserHome}}/Library/Logs/{{SvcName}}/stdout.log</string>
<key>StandardErrorPath</key>
<string>{{UserHome}}/Library/Logs/{{SvcName}}/stderr.log</string>
<key>EnvironmentVariables</key>
<dict>
<key>ACTIONS_RUNNER_SVC</key>
<string>1</string>
</dict>
</dict>
</plist>

View File

@@ -0,0 +1,14 @@
[Unit]
Description={{Description}}
After=network.target
[Service]
ExecStart={{RunnerRoot}}/runsvc.sh
User={{User}}
WorkingDirectory={{RunnerRoot}}
KillMode=process
KillSignal=SIGTERM
TimeoutStopSec=5min
[Install]
WantedBy=multi-user.target

View File

@@ -0,0 +1,135 @@
#!/bin/bash
SVC_NAME="{{SvcNameVar}}"
SVC_DESCRIPTION="{{SvcDescription}}"
user_id=`id -u`
# launchctl should not run as sudo for launch runners
if [ $user_id -eq 0 ]; then
echo "Must not run with sudo"
exit 1
fi
SVC_CMD=$1
RUNNER_ROOT=`pwd`
LAUNCH_PATH="${HOME}/Library/LaunchAgents"
PLIST_PATH="${LAUNCH_PATH}/${SVC_NAME}.plist"
TEMPLATE_PATH=./bin/actions.runner.plist.template
TEMP_PATH=./bin/actions.runner.plist.temp
CONFIG_PATH=.service
function failed()
{
local error=${1:-Undefined error}
echo "Failed: $error" >&2
exit 1
}
if [ ! -f "${TEMPLATE_PATH}" ]; then
failed "Must run from runner root or install is corrupt"
fi
function install()
{
echo "Creating launch runner in ${PLIST_PATH}"
if [ ! -d "${LAUNCH_PATH}" ]; then
mkdir ${LAUNCH_PATH}
fi
if [ -f "${PLIST_PATH}" ]; then
failed "error: exists ${PLIST_PATH}"
fi
if [ -f "${TEMP_PATH}" ]; then
rm "${TEMP_PATH}" || failed "failed to delete ${TEMP_PATH}"
fi
log_path="${HOME}/Library/Logs/${SVC_NAME}"
echo "Creating ${log_path}"
mkdir -p "${log_path}" || failed "failed to create ${log_path}"
echo Creating ${PLIST_PATH}
sed "s/{{User}}/${SUDO_USER:-$USER}/g; s/{{SvcName}}/$SVC_NAME/g; s@{{RunnerRoot}}@${RUNNER_ROOT}@g; s@{{UserHome}}@$HOME@g;" "${TEMPLATE_PATH}" > "${TEMP_PATH}" || failed "failed to create replacement temp file"
mv "${TEMP_PATH}" "${PLIST_PATH}" || failed "failed to copy plist"
# Since we started with sudo, runsvc.sh will be owned by root. Change this to current login user.
echo Creating runsvc.sh
cp ./bin/runsvc.sh ./runsvc.sh || failed "failed to copy runsvc.sh"
chmod u+x ./runsvc.sh || failed "failed to set permission for runsvc.sh"
echo Creating ${CONFIG_PATH}
echo "${PLIST_PATH}" > ${CONFIG_PATH} || failed "failed to create .Service file"
echo "svc install complete"
}
function start()
{
echo "starting ${SVC_NAME}"
launchctl load -w "${PLIST_PATH}" || failed "failed to load ${PLIST_PATH}"
status
}
function stop()
{
echo "stopping ${SVC_NAME}"
launchctl unload "${PLIST_PATH}" || failed "failed to unload ${PLIST_PATH}"
status
}
function uninstall()
{
echo "uninstalling ${SVC_NAME}"
stop
rm "${PLIST_PATH}" || failed "failed to delete ${PLIST_PATH}"
if [ -f "${CONFIG_PATH}" ]; then
rm "${CONFIG_PATH}" || failed "failed to delete ${CONFIG_PATH}"
fi
}
function status()
{
echo "status ${SVC_NAME}:"
if [ -f "${PLIST_PATH}" ]; then
echo
echo "${PLIST_PATH}"
else
echo
echo "not installed"
echo
return
fi
echo
status_out=`launchctl list | grep "${SVC_NAME}"`
if [ ! -z "$status_out" ]; then
echo Started:
echo $status_out
echo
else
echo Stopped
echo
fi
}
function usage()
{
echo
echo Usage:
echo "./svc.sh [install, start, stop, status, uninstall]"
echo
}
case $SVC_CMD in
"install") install;;
"status") status;;
"uninstall") uninstall;;
"start") start;;
"stop") stop;;
*) usage;;
esac
exit 0

View File

@@ -0,0 +1,298 @@
#!/bin/bash
user_id=`id -u`
if [ $user_id -ne 0 ]; then
echo "Need to run with sudo privilege"
exit 1
fi
# Determine OS type
# Debian based OS (Debian, Ubuntu, Linux Mint) has /etc/debian_version
# Fedora based OS (Fedora, Redhat, Centos, Oracle Linux 7) has /etc/redhat-release
# SUSE based OS (OpenSUSE, SUSE Enterprise) has ID_LIKE=suse in /etc/os-release
function print_errormessage()
{
echo "Can't install dotnet core dependencies."
echo "You can manually install all required dependencies based on following documentation"
echo "https://docs.microsoft.com/en-us/dotnet/core/linux-prerequisites?tabs=netcore2x"
}
function print_rhel6message()
{
echo "We did our best effort to install dotnet core dependencies"
echo "However, there are some dependencies which require manual installation"
echo "You can install all remaining required dependencies based on the following documentation"
echo "https://github.com/dotnet/core/blob/master/Documentation/build-and-install-rhel6-prerequisites.md"
}
function print_rhel6errormessage()
{
echo "We couldn't install dotnet core dependencies"
echo "You can manually install all required dependencies based on following documentation"
echo "https://docs.microsoft.com/en-us/dotnet/core/linux-prerequisites?tabs=netcore2x"
echo "In addition, there are some dependencies which require manual installation. Please follow this documentation"
echo "https://github.com/dotnet/core/blob/master/Documentation/build-and-install-rhel6-prerequisites.md"
}
if [ -e /etc/os-release ]
then
echo "--------OS Information--------"
cat /etc/os-release
echo "------------------------------"
if [ -e /etc/debian_version ]
then
echo "The current OS is Debian based"
echo "--------Debian Version--------"
cat /etc/debian_version
echo "------------------------------"
# prefer apt over apt-get
command -v apt
if [ $? -eq 0 ]
then
apt update && apt install -y liblttng-ust0 libkrb5-3 zlib1g
if [ $? -ne 0 ]
then
echo "'apt' failed with exit code '$?'"
print_errormessage
exit 1
fi
# ubuntu 18 uses libcurl4
# ubuntu 14, 16 and other linux use libcurl3
apt install -y libcurl3 || apt install -y libcurl4
if [ $? -ne 0 ]
then
echo "'apt' failed with exit code '$?'"
print_errormessage
exit 1
fi
# debian 9 use libssl1.0.2
# other debian linux use libssl1.0.0
apt install -y libssl1.0.0 || apt install -y libssl1.0.2
if [ $? -ne 0 ]
then
echo "'apt' failed with exit code '$?'"
print_errormessage
exit 1
fi
# libicu version prefer: libicu52 -> libicu55 -> libicu57 -> libicu60
apt install -y libicu52 || apt install -y libicu55 || apt install -y libicu57 || apt install -y libicu60
if [ $? -ne 0 ]
then
echo "'apt' failed with exit code '$?'"
print_errormessage
exit 1
fi
else
command -v apt-get
if [ $? -eq 0 ]
then
apt-get update && apt-get install -y liblttng-ust0 libkrb5-3 zlib1g
if [ $? -ne 0 ]
then
echo "'apt-get' failed with exit code '$?'"
print_errormessage
exit 1
fi
# ubuntu 18 uses libcurl4
# ubuntu 14, 16 and other linux use libcurl3
apt-get install -y libcurl3 || apt-get install -y libcurl4
if [ $? -ne 0 ]
then
echo "'apt-get' failed with exit code '$?'"
print_errormessage
exit 1
fi
# debian 9 use libssl1.0.2
# other debian linux use libssl1.0.0
apt-get install -y libssl1.0.0 || apt install -y libssl1.0.2
if [ $? -ne 0 ]
then
echo "'apt-get' failed with exit code '$?'"
print_errormessage
exit 1
fi
# libicu version prefer: libicu52 -> libicu55 -> libicu57 -> libicu60
apt-get install -y libicu52 || apt install -y libicu55 || apt install -y libicu57 || apt install -y libicu60
if [ $? -ne 0 ]
then
echo "'apt-get' failed with exit code '$?'"
print_errormessage
exit 1
fi
else
echo "Can not find 'apt' or 'apt-get'"
print_errormessage
exit 1
fi
fi
elif [ -e /etc/redhat-release ]
then
echo "The current OS is Fedora based"
echo "--------Redhat Version--------"
cat /etc/redhat-release
echo "------------------------------"
# use dnf on fedora
# use yum on centos and redhat
if [ -e /etc/fedora-release ]
then
command -v dnf
if [ $? -eq 0 ]
then
useCompatSsl=0
grep -i 'fedora release 28' /etc/fedora-release
if [ $? -eq 0 ]
then
useCompatSsl=1
else
grep -i 'fedora release 27' /etc/fedora-release
if [ $? -eq 0 ]
then
useCompatSsl=1
else
grep -i 'fedora release 26' /etc/fedora-release
if [ $? -eq 0 ]
then
useCompatSsl=1
fi
fi
fi
if [ $useCompatSsl -eq 1 ]
then
echo "Use compat-openssl10-devel instead of openssl-devel for Fedora 27/28 (dotnet core requires openssl 1.0.x)"
dnf install -y compat-openssl10
if [ $? -ne 0 ]
then
echo "'dnf' failed with exit code '$?'"
print_errormessage
exit 1
fi
else
dnf install -y openssl-libs
if [ $? -ne 0 ]
then
echo "'dnf' failed with exit code '$?'"
print_errormessage
exit 1
fi
fi
dnf install -y lttng-ust libcurl krb5-libs zlib libicu
if [ $? -ne 0 ]
then
echo "'dnf' failed with exit code '$?'"
print_errormessage
exit 1
fi
else
echo "Can not find 'dnf'"
print_errormessage
exit 1
fi
else
command -v yum
if [ $? -eq 0 ]
then
yum install -y openssl-libs libcurl krb5-libs zlib libicu
if [ $? -ne 0 ]
then
echo "'yum' failed with exit code '$?'"
print_errormessage
exit 1
fi
# install lttng-ust separately since it's not part of offical package repository
yum install -y wget && wget -P /etc/yum.repos.d/ https://packages.efficios.com/repo.files/EfficiOS-RHEL7-x86-64.repo && rpmkeys --import https://packages.efficios.com/rhel/repo.key && yum updateinfo && yum install -y lttng-ust
if [ $? -ne 0 ]
then
echo "'lttng-ust' installation failed with exit code '$?'"
print_errormessage
exit 1
fi
else
echo "Can not find 'yum'"
print_errormessage
exit 1
fi
fi
else
# we might on OpenSUSE
OSTYPE=$(grep ID_LIKE /etc/os-release | cut -f2 -d=)
echo $OSTYPE
if [ $OSTYPE == '"suse"' ]
then
echo "The current OS is SUSE based"
command -v zypper
if [ $? -eq 0 ]
then
zypper -n install lttng-ust libopenssl1_0_0 libcurl4 krb5 zlib libicu52_1
if [ $? -ne 0 ]
then
echo "'zypper' failed with exit code '$?'"
print_errormessage
exit 1
fi
else
echo "Can not find 'zypper'"
print_errormessage
exit 1
fi
else
echo "Can't detect current OS type based on /etc/os-release."
print_errormessage
exit 1
fi
fi
elif [ -e /etc/redhat-release ]
# RHEL6 doesn't have an os-release file defined, read redhat-release instead
then
redhatRelease=$(</etc/redhat-release)
if [[ $redhatRelease == "CentOS release 6."* || $redhatRelease == "Red Hat Enterprise Linux Server release 6."* ]]
then
echo "The current OS is Red Hat Enterprise Linux 6 or Centos 6"
# Install known dependencies, as a best effort.
# The remaining dependencies are covered by the GitHub doc that will be shown by `print_rhel6message`
command -v yum
if [ $? -eq 0 ]
then
yum install -y openssl krb5-libs zlib
if [ $? -ne 0 ]
then
echo "'yum' failed with exit code '$?'"
print_rhel6errormessage
exit 1
fi
else
echo "Can not find 'yum'"
print_rhel6errormessage
exit 1
fi
print_rhel6message
exit 1
else
echo "Unknown RHEL OS version"
print_errormessage
exit 1
fi
else
echo "Unknown OS version"
print_errormessage
exit 1
fi
echo "-----------------------------"
echo " Finish Install Dependencies"
echo "-----------------------------"

20
src/Misc/layoutbin/runsvc.sh Executable file
View File

@@ -0,0 +1,20 @@
#!/bin/bash
# convert SIGTERM signal to SIGINT
# for more info on how to propagate SIGTERM to a child process see: http://veithen.github.io/2014/11/16/sigterm-propagation.html
trap 'kill -INT $PID' TERM INT
if [ -f ".path" ]; then
# configure
export PATH=`cat .path`
echo ".path=${PATH}"
fi
# insert anything to setup env when running as a service
# run the host process which keep the listener alive
./externals/node12/bin/node ./bin/RunnerService.js &
PID=$!
wait $PID
trap - TERM INT
wait $PID

View File

@@ -0,0 +1,143 @@
#!/bin/bash
SVC_NAME="{{SvcNameVar}}"
SVC_DESCRIPTION="{{SvcDescription}}"
SVC_CMD=$1
arg_2=${2}
RUNNER_ROOT=`pwd`
UNIT_PATH=/etc/systemd/system/${SVC_NAME}
TEMPLATE_PATH=./bin/actions.runner.service.template
TEMP_PATH=./bin/actions.runner.service.temp
CONFIG_PATH=.service
user_id=`id -u`
# systemctl must run as sudo
# this script is a convenience wrapper around systemctl
if [ $user_id -ne 0 ]; then
echo "Must run as sudo"
exit 1
fi
function failed()
{
local error=${1:-Undefined error}
echo "Failed: $error" >&2
exit 1
}
if [ ! -f "${TEMPLATE_PATH}" ]; then
failed "Must run from runner root or install is corrupt"
fi
#check if we run as root
if [[ $(id -u) != "0" ]]; then
echo "Failed: This script requires to run with sudo." >&2
exit 1
fi
function install()
{
echo "Creating launch runner in ${UNIT_PATH}"
if [ -f "${UNIT_PATH}" ]; then
failed "error: exists ${UNIT_PATH}"
fi
if [ -f "${TEMP_PATH}" ]; then
rm "${TEMP_PATH}" || failed "failed to delete ${TEMP_PATH}"
fi
# can optionally use username supplied
run_as_user=${arg_2:-$SUDO_USER}
echo "Run as user: ${run_as_user}"
run_as_uid=$(id -u ${run_as_user}) || failed "User does not exist"
echo "Run as uid: ${run_as_uid}"
run_as_gid=$(id -g ${run_as_user}) || failed "Group not available"
echo "gid: ${run_as_gid}"
sed "s/{{User}}/${run_as_user}/g; s/{{Description}}/$(echo ${SVC_DESCRIPTION} | sed -e 's/[\/&]/\\&/g')/g; s/{{RunnerRoot}}/$(echo ${RUNNER_ROOT} | sed -e 's/[\/&]/\\&/g')/g;" "${TEMPLATE_PATH}" > "${TEMP_PATH}" || failed "failed to create replacement temp file"
mv "${TEMP_PATH}" "${UNIT_PATH}" || failed "failed to copy unit file"
# unit file should not be executable and world writable
chmod 664 ${UNIT_PATH} || failed "failed to set permissions on ${UNIT_PATH}"
systemctl daemon-reload || failed "failed to reload daemons"
# Since we started with sudo, runsvc.sh will be owned by root. Change this to current login user.
cp ./bin/runsvc.sh ./runsvc.sh || failed "failed to copy runsvc.sh"
chown ${run_as_uid}:${run_as_gid} ./runsvc.sh || failed "failed to set owner for runsvc.sh"
chmod 755 ./runsvc.sh || failed "failed to set permission for runsvc.sh"
systemctl enable ${SVC_NAME} || failed "failed to enable ${SVC_NAME}"
echo "${SVC_NAME}" > ${CONFIG_PATH} || failed "failed to create .service file"
chown ${run_as_uid}:${run_as_gid} ${CONFIG_PATH} || failed "failed to set permission for ${CONFIG_PATH}"
}
function start()
{
systemctl start ${SVC_NAME} || failed "failed to start ${SVC_NAME}"
status
}
function stop()
{
systemctl stop ${SVC_NAME} || failed "failed to stop ${SVC_NAME}"
status
}
function uninstall()
{
stop
systemctl disable ${SVC_NAME} || failed "failed to disable ${SVC_NAME}"
rm "${UNIT_PATH}" || failed "failed to delete ${UNIT_PATH}"
if [ -f "${CONFIG_PATH}" ]; then
rm "${CONFIG_PATH}" || failed "failed to delete ${CONFIG_PATH}"
fi
systemctl daemon-reload || failed "failed to reload daemons"
}
function status()
{
if [ -f "${UNIT_PATH}" ]; then
echo
echo "${UNIT_PATH}"
else
echo
echo "not installed"
echo
return
fi
systemctl --no-pager status ${SVC_NAME}
}
function usage()
{
echo
echo Usage:
echo "./svc.sh [install, start, stop, status, uninstall]"
echo "Commands:"
echo " install [user]: Install runner service as Root or specified user."
echo " start: Manually start the runner service."
echo " stop: Manually stop the runner service."
echo " status: Display status of runner service."
echo " uninstall: Uninstall runner service."
echo
}
case $SVC_CMD in
"install") install;;
"status") status;;
"uninstall") uninstall;;
"start") start;;
"stop") stop;;
"status") status;;
*) usage;;
esac
exit 0

View File

@@ -0,0 +1,143 @@
@echo off
rem runner will replace key words in the template and generate a batch script to run.
rem Keywords:
rem PROCESSID = pid
rem RUNNERPROCESSNAME = Runner.Listener[.exe]
rem ROOTFOLDER = ./
rem EXISTRUNNERVERSION = 2.100.0
rem DOWNLOADRUNNERVERSION = 2.101.0
rem UPDATELOG = _diag/SelfUpdate-UTC.log
rem RESTARTINTERACTIVERUNNER = 0/1
setlocal
set runnerpid=_PROCESS_ID_
set runnerprocessname=_RUNNER_PROCESS_NAME_
set rootfolder=_ROOT_FOLDER_
set existrunnerversion=_EXIST_RUNNER_VERSION_
set downloadrunnerversion=_DOWNLOAD_RUNNER_VERSION_
set logfile=_UPDATE_LOG_
set restartinteractiverunner=_RESTART_INTERACTIVE_RUNNER_
rem log user who run the script
echo [%date% %time%] --------whoami-------- >> "%logfile%" 2>&1
whoami >> "%logfile%" 2>&1
echo [%date% %time%] --------whoami-------- >> "%logfile%" 2>&1
rem wait for runner process to exit.
echo [%date% %time%] Waiting for %runnerprocessname% (%runnerpid%) to complete >> "%logfile%" 2>&1
:loop
tasklist /fi "pid eq %runnerpid%" | find /I "%runnerprocessname%" >> "%logfile%" 2>&1
if ERRORLEVEL 1 (
goto copy
)
echo [%date% %time%] Process %runnerpid% still running, check again after 1 second. >> "%logfile%" 2>&1
ping -n 2 127.0.0.1 >nul
goto loop
rem start re-organize folders
:copy
echo [%date% %time%] Process %runnerpid% finished running >> "%logfile%" 2>&1
echo [%date% %time%] Sleep 1 more second to make sure process exited >> "%logfile%" 2>&1
ping -n 2 127.0.0.1 >nul
echo [%date% %time%] Re-organize folders >> "%logfile%" 2>&1
rem the folder structure under runner root will be
rem ./bin -> bin.2.100.0 (junction folder)
rem ./externals -> externals.2.100.0 (junction folder)
rem ./bin.2.100.0
rem ./externals.2.100.0
rem ./bin.2.99.0
rem ./externals.2.99.0
rem by using the juction folder we can avoid file in use problem.
rem if the bin/externals junction point already exist, we just need to delete the juction point then re-create to point to new bin/externals folder.
rem if the bin/externals still are real folders, we need to rename the existing folder to bin.version format then create junction point to new bin/externals folder.
rem check bin folder
rem we do findstr /C:" bin" since in migration mode, we create a junction folder from runner to bin.
rem as result, dir /AL | findstr "bin" will return the runner folder. output looks like (07/27/2016 05:21 PM <JUNCTION> runner [E:\bin])
dir "%rootfolder%" /AL 2>&1 | findstr /C:" bin" >> "%logfile%" 2>&1
if ERRORLEVEL 1 (
rem return code 1 means it can't find a bin folder that is a junction folder
rem so we need to move the current bin folder to bin.2.99.0 folder.
echo [%date% %time%] move "%rootfolder%\bin" "%rootfolder%\bin.%existrunnerversion%" >> "%logfile%" 2>&1
move "%rootfolder%\bin" "%rootfolder%\bin.%existrunnerversion%" >> "%logfile%" 2>&1
if ERRORLEVEL 1 (
echo [%date% %time%] Can't move "%rootfolder%\bin" to "%rootfolder%\bin.%existrunnerversion%" >> "%logfile%" 2>&1
goto fail
)
) else (
rem otherwise it find a bin folder that is a junction folder
rem we just need to delete the junction point.
echo [%date% %time%] Delete existing junction bin folder >> "%logfile%" 2>&1
rmdir "%rootfolder%\bin" >> "%logfile%" 2>&1
if ERRORLEVEL 1 (
echo [%date% %time%] Can't delete existing junction bin folder >> "%logfile%" 2>&1
goto fail
)
)
rem check externals folder
dir "%rootfolder%" /AL 2>&1 | findstr "externals" >> "%logfile%" 2>&1
if ERRORLEVEL 1 (
rem return code 1 means it can't find a externals folder that is a junction folder
rem so we need to move the current externals folder to externals.2.99.0 folder.
echo [%date% %time%] move "%rootfolder%\externals" "%rootfolder%\externals.%existrunnerversion%" >> "%logfile%" 2>&1
move "%rootfolder%\externals" "%rootfolder%\externals.%existrunnerversion%" >> "%logfile%" 2>&1
if ERRORLEVEL 1 (
echo [%date% %time%] Can't move "%rootfolder%\externals" to "%rootfolder%\externals.%existrunnerversion%" >> "%logfile%" 2>&1
goto fail
)
) else (
rem otherwise it find a externals folder that is a junction folder
rem we just need to delete the junction point.
echo [%date% %time%] Delete existing junction externals folder >> "%logfile%" 2>&1
rmdir "%rootfolder%\externals" >> "%logfile%" 2>&1
if ERRORLEVEL 1 (
echo [%date% %time%] Can't delete existing junction externals folder >> "%logfile%" 2>&1
goto fail
)
)
rem create junction bin folder
echo [%date% %time%] Create junction bin folder >> "%logfile%" 2>&1
mklink /J "%rootfolder%\bin" "%rootfolder%\bin.%downloadrunnerversion%" >> "%logfile%" 2>&1
if ERRORLEVEL 1 (
echo [%date% %time%] Can't create junction bin folder >> "%logfile%" 2>&1
goto fail
)
rem create junction externals folder
echo [%date% %time%] Create junction externals folder >> "%logfile%" 2>&1
mklink /J "%rootfolder%\externals" "%rootfolder%\externals.%downloadrunnerversion%" >> "%logfile%" 2>&1
if ERRORLEVEL 1 (
echo [%date% %time%] Can't create junction externals folder >> "%logfile%" 2>&1
goto fail
)
echo [%date% %time%] Update succeed >> "%logfile%" 2>&1
rem rename the update log file with %logfile%.succeed/.failed/succeedneedrestart
rem runner service host can base on the log file name determin the result of the runner update
echo [%date% %time%] Rename "%logfile%" to be "%logfile%.succeed" >> "%logfile%" 2>&1
move "%logfile%" "%logfile%.succeed" >nul
rem restart interactive runner if needed
if %restartinteractiverunner% equ 1 (
echo [%date% %time%] Restart interactive runner >> "%logfile%.succeed" 2>&1
endlocal
start "Actions Runner" cmd.exe /k "_ROOT_FOLDER_\run.cmd"
) else (
endlocal
)
goto :eof
:fail
echo [%date% %time%] Rename "%logfile%" to be "%logfile%.failed" >> "%logfile%" 2>&1
move "%logfile%" "%logfile%.failed" >nul
goto :eof

View File

@@ -0,0 +1,133 @@
#!/bin/bash
# runner will replace key words in the template and generate a batch script to run.
# Keywords:
# PROCESSID = pid
# RUNNERPROCESSNAME = Runner.Listener[.exe]
# ROOTFOLDER = ./
# EXISTRUNNERVERSION = 2.100.0
# DOWNLOADRUNNERVERSION = 2.101.0
# UPDATELOG = _diag/SelfUpdate-UTC.log
# RESTARTINTERACTIVERUNNER = 0/1
runnerpid=_PROCESS_ID_
runnerprocessname=_RUNNER_PROCESS_NAME_
rootfolder="_ROOT_FOLDER_"
existrunnerversion=_EXIST_RUNNER_VERSION_
downloadrunnerversion=_DOWNLOAD_RUNNER_VERSION_
logfile="_UPDATE_LOG_"
restartinteractiverunner=_RESTART_INTERACTIVE_RUNNER_
# log user who run the script
date "+[%F %T-%4N] --------whoami--------" >> "$logfile" 2>&1
whoami >> "$logfile" 2>&1
date "+[%F %T-%4N] --------whoami--------" >> "$logfile" 2>&1
# wait for runner process to exit.
date "+[%F %T-%4N] Waiting for $runnerprocessname ($runnerpid) to complete" >> "$logfile" 2>&1
while [ -e /proc/$runnerpid ]
do
date "+[%F %T-%4N] Process $runnerpid still running" >> "$logfile" 2>&1
ping -c 2 127.0.0.1 >nul
done
date "+[%F %T-%4N] Process $runnerpid finished running" >> "$logfile" 2>&1
# start re-organize folders
date "+[%F %T-%4N] Sleep 1 more second to make sure process exited" >> "$logfile" 2>&1
ping -c 2 127.0.0.1 >nul
# the folder structure under runner root will be
# ./bin -> bin.2.100.0 (junction folder)
# ./externals -> externals.2.100.0 (junction folder)
# ./bin.2.100.0
# ./externals.2.100.0
# ./bin.2.99.0
# ./externals.2.99.0
# by using the juction folder we can avoid file in use problem.
# if the bin/externals junction point already exist, we just need to delete the juction point then re-create to point to new bin/externals folder.
# if the bin/externals still are real folders, we need to rename the existing folder to bin.version format then create junction point to new bin/externals folder.
# check bin folder
if [[ -L "$rootfolder/bin" && -d "$rootfolder/bin" ]]
then
# return code 0 means it find a bin folder that is a junction folder
# we just need to delete the junction point.
date "+[%F %T-%4N] Delete existing junction bin folder" >> "$logfile"
rm "$rootfolder/bin" >> "$logfile"
if [ $? -ne 0 ]
then
date "+[%F %T-%4N] Can't delete existing junction bin folder" >> "$logfile"
mv -fv "$logfile" "$logfile.failed"
exit 1
fi
else
# otherwise, we need to move the current bin folder to bin.2.99.0 folder.
date "+[%F %T-%4N] move $rootfolder/bin $rootfolder/bin.$existrunnerversion" >> "$logfile" 2>&1
mv -fv "$rootfolder/bin" "$rootfolder/bin.$existrunnerversion" >> "$logfile" 2>&1
if [ $? -ne 0 ]
then
date "+[%F %T-%4N] Can't move $rootfolder/bin to $rootfolder/bin.$existrunnerversion" >> "$logfile" 2>&1
mv -fv "$logfile" "$logfile.failed"
exit 1
fi
fi
# check externals folder
if [[ -L "$rootfolder/externals" && -d "$rootfolder/externals" ]]
then
# the externals folder is already a junction folder
# we just need to delete the junction point.
date "+[%F %T-%4N] Delete existing junction externals folder" >> "$logfile"
rm "$rootfolder/externals" >> "$logfile"
if [ $? -ne 0 ]
then
date "+[%F %T-%4N] Can't delete existing junction externals folder" >> "$logfile"
mv -fv "$logfile" "$logfile.failed"
exit 1
fi
else
# otherwise, we need to move the current externals folder to externals.2.99.0 folder.
date "+[%F %T-%4N] move $rootfolder/externals $rootfolder/externals.$existrunnerversion" >> "$logfile" 2>&1
mv -fv "$rootfolder/externals" "$rootfolder/externals.$existrunnerversion" >> "$logfile" 2>&1
if [ $? -ne 0 ]
then
date "+[%F %T-%4N] Can't move $rootfolder/externals to $rootfolder/externals.$existrunnerversion" >> "$logfile" 2>&1
mv -fv "$logfile" "$logfile.failed"
exit 1
fi
fi
# create junction bin folder
date "+[%F %T-%4N] Create junction bin folder" >> "$logfile" 2>&1
ln -s "$rootfolder/bin.$downloadrunnerversion" "$rootfolder/bin" >> "$logfile" 2>&1
if [ $? -ne 0 ]
then
date "+[%F %T-%4N] Can't create junction bin folder" >> "$logfile" 2>&1
mv -fv "$logfile" "$logfile.failed"
exit 1
fi
# create junction externals folder
date "+[%F %T-%4N] Create junction externals folder" >> "$logfile" 2>&1
ln -s "$rootfolder/externals.$downloadrunnerversion" "$rootfolder/externals" >> "$logfile" 2>&1
if [ $? -ne 0 ]
then
date "+[%F %T-%4N] Can't create junction externals folder" >> "$logfile" 2>&1
mv -fv "$logfile" "$logfile.failed"
exit 1
fi
date "+[%F %T-%4N] Update succeed" >> "$logfile"
# rename the update log file with %logfile%.succeed/.failed/succeedneedrestart
# runner service host can base on the log file name determin the result of the runner update
date "+[%F %T-%4N] Rename $logfile to be $logfile.succeed" >> "$logfile" 2>&1
mv -fv "$logfile" "$logfile.succeed" >> "$logfile" 2>&1
# restart interactive runner if needed
if [ $restartinteractiverunner -ne 0 ]
then
date "+[%F %T-%4N] Restarting interactive runner" >> "$logfile.succeed" 2>&1
"$rootfolder/run.sh" &
fi

View File

@@ -0,0 +1,26 @@
@echo off
rem ********************************************************************************
rem Unblock specific files.
rem ********************************************************************************
setlocal
if defined VERBOSE_ARG (
set VERBOSE_ARG='Continue'
) else (
set VERBOSE_ARG='SilentlyContinue'
)
rem Unblock files in the root of the layout folder. E.g. .cmd files.
powershell.exe -NoLogo -Sta -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command "$VerbosePreference = %VERBOSE_ARG% ; Get-ChildItem -LiteralPath '%~dp0' | ForEach-Object { Write-Verbose ('Unblock: {0}' -f $_.FullName) ; $_ } | Unblock-File | Out-Null"
if /i "%~1" equ "remove" (
rem ********************************************************************************
rem Unconfigure the runner.
rem ********************************************************************************
"%~dp0bin\Runner.Listener.exe" %*
) else (
rem ********************************************************************************
rem Configure the runner.
rem ********************************************************************************
"%~dp0bin\Runner.Listener.exe" configure %*
)

86
src/Misc/layoutroot/config.sh Executable file
View File

@@ -0,0 +1,86 @@
#!/bin/bash
user_id=`id -u`
# we want to snapshot the environment of the config user
if [ $user_id -eq 0 -a -z "$AGENT_ALLOW_RUNASROOT" ]; then
echo "Must not run with sudo"
exit 1
fi
# Check dotnet core 2.1 dependencies for Linux
if [[ (`uname` == "Linux") ]]
then
command -v ldd > /dev/null
if [ $? -ne 0 ]
then
echo "Can not find 'ldd'. Please install 'ldd' and try again."
exit 1
fi
ldd ./bin/libcoreclr.so | grep 'not found'
if [ $? -eq 0 ]; then
echo "Dependencies is missing for Dotnet Core 2.1"
echo "Execute ./bin/installdependencies.sh to install any missing Dotnet Core 2.1 dependencies."
exit 1
fi
ldd ./bin/System.Security.Cryptography.Native.OpenSsl.so | grep 'not found'
if [ $? -eq 0 ]; then
echo "Dependencies is missing for Dotnet Core 2.1"
echo "Execute ./bin/installdependencies.sh to install any missing Dotnet Core 2.1 dependencies."
exit 1
fi
ldd ./bin/System.IO.Compression.Native.so | grep 'not found'
if [ $? -eq 0 ]; then
echo "Dependencies is missing for Dotnet Core 2.1"
echo "Execute ./bin/installdependencies.sh to install any missing Dotnet Core 2.1 dependencies."
exit 1
fi
ldd ./bin/System.Net.Http.Native.so | grep 'not found'
if [ $? -eq 0 ]; then
echo "Dependencies is missing for Dotnet Core 2.1"
echo "Execute ./bin/installdependencies.sh to install any missing Dotnet Core 2.1 dependencies."
exit 1
fi
if ! [ -x "$(command -v ldconfig)" ]; then
LDCONFIG_COMMAND="/sbin/ldconfig"
if ! [ -x "$LDCONFIG_COMMAND" ]; then
echo "Can not find 'ldconfig' in PATH and '/sbin/ldconfig' doesn't exists either. Please install 'ldconfig' and try again."
exit 1
fi
else
LDCONFIG_COMMAND="ldconfig"
fi
libpath=${LD_LIBRARY_PATH:-}
$LDCONFIG_COMMAND -NXv ${libpath//:/} 2>&1 | grep libicu >/dev/null 2>&1
if [ $? -ne 0 ]; then
echo "Libicu's dependencies is missing for Dotnet Core 2.1"
echo "Execute ./bin/installdependencies.sh to install any missing Dotnet Core 2.1 dependencies."
exit 1
fi
fi
# Change directory to the script root directory
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
cd $DIR
source ./env.sh
shopt -s nocasematch
if [[ "$1" == "remove" ]]; then
./bin/Runner.Listener "$@"
else
./bin/Runner.Listener configure "$@"
fi

44
src/Misc/layoutroot/env.sh Executable file
View File

@@ -0,0 +1,44 @@
#!/bin/bash
varCheckList=(
'LANG'
'JAVA_HOME'
'ANT_HOME'
'M2_HOME'
'ANDROID_HOME'
'GRADLE_HOME'
'NVM_BIN'
'NVM_PATH'
'VSTS_HTTP_PROXY'
'VSTS_HTTP_PROXY_USERNAME'
'VSTS_HTTP_PROXY_PASSWORD'
'LD_LIBRARY_PATH'
'PERL5LIB'
)
envContents=""
if [ -f ".env" ]; then
envContents=`cat .env`
else
touch .env
fi
function writeVar()
{
checkVar="$1"
checkDelim="${1}="
if test "${envContents#*$checkDelim}" = "$envContents"
then
if [ ! -z "${!checkVar}" ]; then
echo "${checkVar}=${!checkVar}">>.env
fi
fi
}
echo $PATH>.path
for var_name in ${varCheckList[@]}
do
writeVar "${var_name}"
done

View File

@@ -0,0 +1,33 @@
@echo off
rem ********************************************************************************
rem Unblock specific files.
rem ********************************************************************************
setlocal
if defined VERBOSE_ARG (
set VERBOSE_ARG='Continue'
) else (
set VERBOSE_ARG='SilentlyContinue'
)
rem Unblock files in the root of the layout folder. E.g. .cmd files.
powershell.exe -NoLogo -Sta -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command "$VerbosePreference = %VERBOSE_ARG% ; Get-ChildItem -LiteralPath '%~dp0' | ForEach-Object { Write-Verbose ('Unblock: {0}' -f $_.FullName) ; $_ } | Unblock-File | Out-Null"
if /i "%~1" equ "localRun" (
rem ********************************************************************************
rem Local run.
rem ********************************************************************************
"%~dp0bin\Runner.Listener.exe" %*
) else (
rem ********************************************************************************
rem Run.
rem ********************************************************************************
"%~dp0bin\Runner.Listener.exe" run %*
rem Return code 4 means the run once runner received an update message.
rem Sleep 5 seconds to wait for the update process finish and run the runner again.
if ERRORLEVEL 4 (
timeout /t 5 /nobreak > NUL
"%~dp0bin\Runner.Listener.exe" run %*
)
)

51
src/Misc/layoutroot/run.sh Executable file
View File

@@ -0,0 +1,51 @@
#!/bin/bash
# Validate not sudo
user_id=`id -u`
if [ $user_id -eq 0 -a -z "$AGENT_ALLOW_RUNASROOT" ]; then
echo "Must not run interactively with sudo"
exit 1
fi
# Change directory to the script root directory
# https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
# Do not "cd $DIR". For localRun, the current directory is expected to be the repo location on disk.
# Run
shopt -s nocasematch
if [[ "$1" == "localRun" ]]; then
"$DIR"/bin/Runner.Listener $*
else
"$DIR"/bin/Runner.Listener run $*
# Return code 4 means the run once agent received an update message.
# Sleep 5 seconds to wait for the update process finish and run the agent again.
returnCode=$?
if [[ $returnCode == 4 ]]; then
if [ ! -x "$(command -v sleep)" ]; then
if [ ! -x "$(command -v ping)" ]; then
COUNT="0"
while [[ $COUNT != 5000 ]]; do
echo "SLEEP" >nul
COUNT=$[$COUNT+1]
done
else
ping -n 5 127.0.0.1 >nul
fi
else
sleep 5 >nul
fi
"$DIR"/bin/Runner.Listener run $*
else
exit $returnCode
fi
fi

10
src/NuGet.Config Normal file
View File

@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<!--To inherit the global NuGet package sources remove the <clear/> line below -->
<clear />
<add key="dotnet-core" value="https://www.myget.org/F/dotnet-core/api/v3/index.json" />
<add key="dotnet-buildtools" value="https://www.myget.org/F/dotnet-buildtools/api/v3/index.json" />
<add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
</packageSources>
</configuration>

View File

@@ -0,0 +1,253 @@
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using System;
using System.Collections.Generic;
namespace GitHub.Runner.Common
{
public sealed class ActionCommand
{
private static readonly EscapeMapping[] _escapeMappings = new[]
{
new EscapeMapping(token: "%", replacement: "%25"),
new EscapeMapping(token: ";", replacement: "%3B"),
new EscapeMapping(token: "\r", replacement: "%0D"),
new EscapeMapping(token: "\n", replacement: "%0A"),
new EscapeMapping(token: "]", replacement: "%5D"),
};
private static readonly EscapeMapping[] _escapeDataMappings = new[]
{
new EscapeMapping(token: "\r", replacement: "%0D"),
new EscapeMapping(token: "\n", replacement: "%0A"),
};
private static readonly EscapeMapping[] _escapePropertyMappings = new[]
{
new EscapeMapping(token: "%", replacement: "%25"),
new EscapeMapping(token: "\r", replacement: "%0D"),
new EscapeMapping(token: "\n", replacement: "%0A"),
new EscapeMapping(token: ":", replacement: "%3A"),
new EscapeMapping(token: ",", replacement: "%2C"),
};
private readonly Dictionary<string, string> _properties = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
public const string Prefix = "##[";
public const string _commandKey = "::";
public ActionCommand(string command)
{
ArgUtil.NotNullOrEmpty(command, nameof(command));
Command = command;
}
public string Command { get; }
public Dictionary<string, string> Properties => _properties;
public string Data { get; set; }
public static bool TryParseV2(string message, HashSet<string> registeredCommands, out ActionCommand command)
{
command = null;
if (string.IsNullOrEmpty(message))
{
return false;
}
try
{
// the message needs to start with the keyword after trim leading space.
message = message.TrimStart();
if (!message.StartsWith(_commandKey))
{
return false;
}
// Get the index of the separator between the command info and the data.
int endIndex = message.IndexOf(_commandKey, _commandKey.Length);
if (endIndex < 0)
{
return false;
}
// Get the command info (command and properties).
int cmdIndex = _commandKey.Length;
string cmdInfo = message.Substring(cmdIndex, endIndex - cmdIndex);
// Get the command name
int spaceIndex = cmdInfo.IndexOf(' ');
string commandName =
spaceIndex < 0
? cmdInfo
: cmdInfo.Substring(0, spaceIndex);
if (registeredCommands.Contains(commandName))
{
// Initialize the command.
command = new ActionCommand(commandName);
}
else
{
return false;
}
// Set the properties.
if (spaceIndex > 0)
{
string propertiesStr = cmdInfo.Substring(spaceIndex + 1).Trim();
string[] splitProperties = propertiesStr.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string propertyStr in splitProperties)
{
string[] pair = propertyStr.Split(new[] { '=' }, count: 2, options: StringSplitOptions.RemoveEmptyEntries);
if (pair.Length == 2)
{
command.Properties[pair[0]] = UnescapeProperty(pair[1]);
}
}
}
command.Data = UnescapeData(message.Substring(endIndex + _commandKey.Length));
return true;
}
catch
{
command = null;
return false;
}
}
public static bool TryParse(string message, HashSet<string> registeredCommands, out ActionCommand command)
{
command = null;
if (string.IsNullOrEmpty(message))
{
return false;
}
try
{
// Get the index of the prefix.
int prefixIndex = message.IndexOf(Prefix);
if (prefixIndex < 0)
{
return false;
}
// Get the index of the separator between the command info and the data.
int rbIndex = message.IndexOf(']', prefixIndex);
if (rbIndex < 0)
{
return false;
}
// Get the command info (command and properties).
int cmdIndex = prefixIndex + Prefix.Length;
string cmdInfo = message.Substring(cmdIndex, rbIndex - cmdIndex);
// Get the command name
int spaceIndex = cmdInfo.IndexOf(' ');
string commandName =
spaceIndex < 0
? cmdInfo
: cmdInfo.Substring(0, spaceIndex);
if (registeredCommands.Contains(commandName))
{
// Initialize the command.
command = new ActionCommand(commandName);
}
else
{
return false;
}
// Set the properties.
if (spaceIndex > 0)
{
string propertiesStr = cmdInfo.Substring(spaceIndex + 1);
string[] splitProperties = propertiesStr.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries);
foreach (string propertyStr in splitProperties)
{
string[] pair = propertyStr.Split(new[] { '=' }, count: 2, options: StringSplitOptions.RemoveEmptyEntries);
if (pair.Length == 2)
{
command.Properties[pair[0]] = Unescape(pair[1]);
}
}
}
command.Data = Unescape(message.Substring(rbIndex + 1));
return true;
}
catch
{
command = null;
return false;
}
}
private static string Unescape(string escaped)
{
if (string.IsNullOrEmpty(escaped))
{
return string.Empty;
}
string unescaped = escaped;
foreach (EscapeMapping mapping in _escapeMappings)
{
unescaped = unescaped.Replace(mapping.Replacement, mapping.Token);
}
return unescaped;
}
private static string UnescapeProperty(string escaped)
{
if (string.IsNullOrEmpty(escaped))
{
return string.Empty;
}
string unescaped = escaped;
foreach (EscapeMapping mapping in _escapePropertyMappings)
{
unescaped = unescaped.Replace(mapping.Replacement, mapping.Token);
}
return unescaped;
}
private static string UnescapeData(string escaped)
{
if (string.IsNullOrEmpty(escaped))
{
return string.Empty;
}
string unescaped = escaped;
foreach (EscapeMapping mapping in _escapeDataMappings)
{
unescaped = unescaped.Replace(mapping.Replacement, mapping.Token);
}
return unescaped;
}
private sealed class EscapeMapping
{
public string Replacement { get; }
public string Token { get; }
public EscapeMapping(string token, string replacement)
{
ArgUtil.NotNullOrEmpty(token, nameof(token));
ArgUtil.NotNullOrEmpty(replacement, nameof(replacement));
Token = token;
Replacement = replacement;
}
}
}
}

View File

@@ -0,0 +1,15 @@
using System;
namespace GitHub.Runner.Common
{
public enum ActionResult
{
Success = 0,
Failure = 1,
Cancelled = 2,
Skipped = 3
}
}

View File

@@ -0,0 +1,33 @@
using System.Threading;
using System.Threading.Tasks;
namespace GitHub.Runner.Common
{
//Stephen Toub: http://blogs.msdn.com/b/pfxteam/archive/2012/02/11/10266920.aspx
public class AsyncManualResetEvent
{
private volatile TaskCompletionSource<bool> m_tcs = new TaskCompletionSource<bool>();
public Task WaitAsync() { return m_tcs.Task; }
public void Set()
{
var tcs = m_tcs;
Task.Factory.StartNew(s => ((TaskCompletionSource<bool>)s).TrySetResult(true),
tcs, CancellationToken.None, TaskCreationOptions.PreferFairness, TaskScheduler.Default);
tcs.Task.Wait();
}
public void Reset()
{
while (true)
{
var tcs = m_tcs;
if (!tcs.Task.IsCompleted ||
Interlocked.CompareExchange(ref m_tcs, new TaskCompletionSource<bool>(), tcs) == tcs)
return;
}
}
}
}

View File

@@ -0,0 +1,73 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common.Capabilities
{
[ServiceLocator(Default = typeof(CapabilitiesManager))]
public interface ICapabilitiesManager : IRunnerService
{
Task<Dictionary<string, string>> GetCapabilitiesAsync(RunnerSettings settings, CancellationToken token);
}
public sealed class CapabilitiesManager : RunnerService, ICapabilitiesManager
{
public async Task<Dictionary<string, string>> GetCapabilitiesAsync(RunnerSettings settings, CancellationToken cancellationToken)
{
Trace.Entering();
ArgUtil.NotNull(settings, nameof(settings));
// Initialize a dictionary of capabilities.
var capabilities = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
if (settings.SkipCapabilitiesScan)
{
Trace.Info("Skip capabilities scan.");
return capabilities;
}
// Get the providers.
var extensionManager = HostContext.GetService<IExtensionManager>();
IEnumerable<ICapabilitiesProvider> providers =
extensionManager
.GetExtensions<ICapabilitiesProvider>()
?.OrderBy(x => x.Order);
// Add each capability returned from each provider.
foreach (ICapabilitiesProvider provider in providers ?? new ICapabilitiesProvider[0])
{
foreach (Capability capability in await provider.GetCapabilitiesAsync(settings, cancellationToken) ?? new List<Capability>())
{
// Make sure we mask secrets in capabilities values.
capabilities[capability.Name] = HostContext.SecretMasker.MaskSecrets(capability.Value);
}
}
return capabilities;
}
}
public interface ICapabilitiesProvider : IExtension
{
int Order { get; }
Task<List<Capability>> GetCapabilitiesAsync(RunnerSettings settings, CancellationToken cancellationToken);
}
public sealed class Capability
{
public string Name { get; }
public string Value { get; }
public Capability(string name, string value)
{
ArgUtil.NotNullOrEmpty(name, nameof(name));
Name = name;
Value = value ?? string.Empty;
}
}
}

View File

@@ -0,0 +1,86 @@
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using Microsoft.Win32;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace GitHub.Runner.Common.Capabilities
{
public sealed class RunnerCapabilitiesProvider : RunnerService, ICapabilitiesProvider
{
public Type ExtensionType => typeof(ICapabilitiesProvider);
public int Order => 99; // Process last to override prior.
public Task<List<Capability>> GetCapabilitiesAsync(RunnerSettings settings, CancellationToken cancellationToken)
{
ArgUtil.NotNull(settings, nameof(settings));
var capabilities = new List<Capability>();
Add(capabilities, "Runner.Name", settings.AgentName ?? string.Empty);
Add(capabilities, "Runner.OS", VarUtil.OS);
Add(capabilities, "Runner.OSArchitecture", VarUtil.OSArchitecture);
#if OS_WINDOWS
Add(capabilities, "Runner.OSVersion", GetOSVersionString());
#endif
Add(capabilities, "InteractiveSession", (HostContext.StartupType != StartupType.Service).ToString());
Add(capabilities, "Runner.Version", BuildConstants.RunnerPackage.Version);
Add(capabilities, "Runner.ComputerName", Environment.MachineName ?? string.Empty);
Add(capabilities, "Runner.HomeDirectory", HostContext.GetDirectory(WellKnownDirectory.Root));
return Task.FromResult(capabilities);
}
private void Add(List<Capability> capabilities, string name, string value)
{
Trace.Info($"Adding '{name}': '{value}'");
capabilities.Add(new Capability(name, value));
}
private object GetHklmValue(string keyName, string valueName)
{
keyName = $@"HKEY_LOCAL_MACHINE\{keyName}";
object value = Registry.GetValue(keyName, valueName, defaultValue: null);
if (object.ReferenceEquals(value, null))
{
Trace.Info($"Key name '{keyName}', value name '{valueName}' is null.");
return null;
}
Trace.Info($"Key name '{keyName}', value name '{valueName}': '{value}'");
return value;
}
private string GetOSVersionString()
{
// Do not use System.Environment.OSVersion.Version to resolve the OS version number.
// It leverages the GetVersionEx function which may report an incorrect version
// depending on the app's manifest. For details, see:
// https://msdn.microsoft.com/library/windows/desktop/ms724451(v=vs.85).aspx
// Attempt to retrieve the major/minor version from the new registry values added in
// in Windows 10.
//
// The registry value "CurrentVersion" is unreliable in Windows 10. It contains the
// value "6.3" instead of "10.0".
object major = GetHklmValue(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion", "CurrentMajorVersionNumber");
object minor = GetHklmValue(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion", "CurrentMinorVersionNumber");
string majorMinorString;
if (major != null && minor != null)
{
majorMinorString = StringUtil.Format("{0}.{1}", major, minor);
}
else
{
// Fallback to the registry value "CurrentVersion".
majorMinorString = GetHklmValue(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion", "CurrentVersion") as string;
}
// Opted to use the registry value "CurrentBuildNumber" over "CurrentBuild". Based on brief
// internet investigation, the only difference appears to be that on Windows XP "CurrentBuild"
// was unreliable and "CurrentBuildNumber" was the correct choice.
string build = GetHklmValue(@"SOFTWARE\Microsoft\Windows NT\CurrentVersion", "CurrentBuildNumber") as string;
return StringUtil.Format("{0}.{1}", majorMinorString, build);
}
}
}

View File

@@ -0,0 +1,128 @@
using GitHub.Runner.Common.Util;
using System;
using System.Collections.Generic;
using GitHub.DistributedTask.Logging;
using GitHub.Runner.Sdk;
//
// Pattern:
// cmd1 cmd2 --arg1 arg1val --aflag --arg2 arg2val
//
namespace GitHub.Runner.Common
{
public sealed class CommandLineParser
{
private ISecretMasker _secretMasker;
private Tracing _trace;
public List<string> Commands { get; }
public HashSet<string> Flags { get; }
public Dictionary<string, string> Args { get; }
public HashSet<string> SecretArgNames { get; }
private bool HasArgs { get; set; }
public CommandLineParser(IHostContext hostContext, string[] secretArgNames)
{
_secretMasker = hostContext.SecretMasker;
_trace = hostContext.GetTrace(nameof(CommandLineParser));
Commands = new List<string>();
Flags = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
Args = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
SecretArgNames = new HashSet<string>(secretArgNames ?? new string[0], StringComparer.OrdinalIgnoreCase);
}
public bool IsCommand(string name)
{
bool result = false;
if (Commands.Count > 0)
{
result = String.Equals(name, Commands[0], StringComparison.CurrentCultureIgnoreCase);
}
return result;
}
public void Parse(string[] args)
{
_trace.Info(nameof(Parse));
ArgUtil.NotNull(args, nameof(args));
_trace.Info("Parsing {0} args", args.Length);
string argScope = null;
foreach (string arg in args)
{
_trace.Info("parsing argument");
HasArgs = HasArgs || arg.StartsWith("--");
_trace.Info("HasArgs: {0}", HasArgs);
if (string.Equals(arg, "/?", StringComparison.Ordinal))
{
Flags.Add("help");
}
else if (!HasArgs)
{
_trace.Info("Adding Command: {0}", arg);
Commands.Add(arg.Trim());
}
else
{
// it's either an arg, an arg value or a flag
if (arg.StartsWith("--") && arg.Length > 2)
{
string argVal = arg.Substring(2);
_trace.Info("arg: {0}", argVal);
// this means two --args in a row which means previous was a flag
if (argScope != null)
{
_trace.Info("Adding flag: {0}", argScope);
Flags.Add(argScope.Trim());
}
argScope = argVal;
}
else if (!arg.StartsWith("-"))
{
// we found a value - check if we're in scope of an arg
if (argScope != null && !Args.ContainsKey(argScope = argScope.Trim()))
{
if (SecretArgNames.Contains(argScope))
{
_secretMasker.AddValue(arg);
}
_trace.Info("Adding option '{0}': '{1}'", argScope, arg);
// ignore duplicates - first wins - below will be val1
// --arg1 val1 --arg1 val1
Args.Add(argScope, arg);
argScope = null;
}
}
else
{
//
// ignoring the second value for an arg (val2 below)
// --arg val1 val2
// ignoring invalid things like empty - and --
// --arg val1 -- --flag
_trace.Info("Ignoring arg");
}
}
}
_trace.Verbose("done parsing arguments");
// handle last arg being a flag
if (argScope != null)
{
Flags.Add(argScope);
}
_trace.Verbose("Exiting parse");
}
}
}

View File

@@ -0,0 +1,252 @@
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using System.IO;
using System.Runtime.Serialization;
using System.Text;
using System.Threading;
namespace GitHub.Runner.Common
{
//
// Settings are persisted in this structure
//
[DataContract]
public sealed class RunnerSettings
{
[DataMember(EmitDefaultValue = false)]
public bool AcceptTeeEula { get; set; }
[DataMember(EmitDefaultValue = false)]
public int AgentId { get; set; }
[DataMember(EmitDefaultValue = false)]
public string AgentName { get; set; }
[DataMember(EmitDefaultValue = false)]
public string NotificationPipeName { get; set; }
[DataMember(EmitDefaultValue = false)]
public string NotificationSocketAddress { get; set; }
[DataMember(EmitDefaultValue = false)]
public bool SkipCapabilitiesScan { get; set; }
[DataMember(EmitDefaultValue = false)]
public bool SkipSessionRecover { get; set; }
[DataMember(EmitDefaultValue = false)]
public int PoolId { get; set; }
[DataMember(EmitDefaultValue = false)]
public string PoolName { get; set; }
[DataMember(EmitDefaultValue = false)]
public string ServerUrl { get; set; }
[DataMember(EmitDefaultValue = false)]
public string GitHubUrl { get; set; }
[DataMember(EmitDefaultValue = false)]
public string WorkFolder { get; set; }
[DataMember(EmitDefaultValue = false)]
public string MonitorSocketAddress { get; set; }
}
[DataContract]
public sealed class RunnerRuntimeOptions
{
#if OS_WINDOWS
[DataMember(EmitDefaultValue = false)]
public bool GitUseSecureChannel { get; set; }
#endif
}
[ServiceLocator(Default = typeof(ConfigurationStore))]
public interface IConfigurationStore : IRunnerService
{
bool IsConfigured();
bool IsServiceConfigured();
bool HasCredentials();
CredentialData GetCredentials();
RunnerSettings GetSettings();
void SaveCredential(CredentialData credential);
void SaveSettings(RunnerSettings settings);
void DeleteCredential();
void DeleteSettings();
RunnerRuntimeOptions GetRunnerRuntimeOptions();
void SaveRunnerRuntimeOptions(RunnerRuntimeOptions options);
void DeleteRunnerRuntimeOptions();
}
public sealed class ConfigurationStore : RunnerService, IConfigurationStore
{
private string _binPath;
private string _configFilePath;
private string _credFilePath;
private string _serviceConfigFilePath;
private string _runtimeOptionsFilePath;
private CredentialData _creds;
private RunnerSettings _settings;
private RunnerRuntimeOptions _runtimeOptions;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
var currentAssemblyLocation = System.Reflection.Assembly.GetEntryAssembly().Location;
Trace.Info("currentAssemblyLocation: {0}", currentAssemblyLocation);
_binPath = HostContext.GetDirectory(WellKnownDirectory.Bin);
Trace.Info("binPath: {0}", _binPath);
RootFolder = HostContext.GetDirectory(WellKnownDirectory.Root);
Trace.Info("RootFolder: {0}", RootFolder);
_configFilePath = hostContext.GetConfigFile(WellKnownConfigFile.Runner);
Trace.Info("ConfigFilePath: {0}", _configFilePath);
_credFilePath = hostContext.GetConfigFile(WellKnownConfigFile.Credentials);
Trace.Info("CredFilePath: {0}", _credFilePath);
_serviceConfigFilePath = hostContext.GetConfigFile(WellKnownConfigFile.Service);
Trace.Info("ServiceConfigFilePath: {0}", _serviceConfigFilePath);
_runtimeOptionsFilePath = hostContext.GetConfigFile(WellKnownConfigFile.Options);
Trace.Info("RuntimeOptionsFilePath: {0}", _runtimeOptionsFilePath);
}
public string RootFolder { get; private set; }
public bool HasCredentials()
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
Trace.Info("HasCredentials()");
bool credsStored = (new FileInfo(_credFilePath)).Exists;
Trace.Info("stored {0}", credsStored);
return credsStored;
}
public bool IsConfigured()
{
Trace.Info("IsConfigured()");
bool configured = HostContext.RunMode == RunMode.Local || (new FileInfo(_configFilePath)).Exists;
Trace.Info("IsConfigured: {0}", configured);
return configured;
}
public bool IsServiceConfigured()
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
Trace.Info("IsServiceConfigured()");
bool serviceConfigured = (new FileInfo(_serviceConfigFilePath)).Exists;
Trace.Info($"IsServiceConfigured: {serviceConfigured}");
return serviceConfigured;
}
public CredentialData GetCredentials()
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
if (_creds == null)
{
_creds = IOUtil.LoadObject<CredentialData>(_credFilePath);
}
return _creds;
}
public RunnerSettings GetSettings()
{
if (_settings == null)
{
RunnerSettings configuredSettings = null;
if (File.Exists(_configFilePath))
{
string json = File.ReadAllText(_configFilePath, Encoding.UTF8);
Trace.Info($"Read setting file: {json.Length} chars");
configuredSettings = StringUtil.ConvertFromJson<RunnerSettings>(json);
}
ArgUtil.NotNull(configuredSettings, nameof(configuredSettings));
_settings = configuredSettings;
}
return _settings;
}
public void SaveCredential(CredentialData credential)
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
Trace.Info("Saving {0} credential @ {1}", credential.Scheme, _credFilePath);
if (File.Exists(_credFilePath))
{
// Delete existing credential file first, since the file is hidden and not able to overwrite.
Trace.Info("Delete exist runner credential file.");
IOUtil.DeleteFile(_credFilePath);
}
IOUtil.SaveObject(credential, _credFilePath);
Trace.Info("Credentials Saved.");
File.SetAttributes(_credFilePath, File.GetAttributes(_credFilePath) | FileAttributes.Hidden);
}
public void SaveSettings(RunnerSettings settings)
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
Trace.Info("Saving runner settings.");
if (File.Exists(_configFilePath))
{
// Delete existing runner settings file first, since the file is hidden and not able to overwrite.
Trace.Info("Delete exist runner settings file.");
IOUtil.DeleteFile(_configFilePath);
}
IOUtil.SaveObject(settings, _configFilePath);
Trace.Info("Settings Saved.");
File.SetAttributes(_configFilePath, File.GetAttributes(_configFilePath) | FileAttributes.Hidden);
}
public void DeleteCredential()
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
IOUtil.Delete(_credFilePath, default(CancellationToken));
}
public void DeleteSettings()
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
IOUtil.Delete(_configFilePath, default(CancellationToken));
}
public RunnerRuntimeOptions GetRunnerRuntimeOptions()
{
if (_runtimeOptions == null && File.Exists(_runtimeOptionsFilePath))
{
_runtimeOptions = IOUtil.LoadObject<RunnerRuntimeOptions>(_runtimeOptionsFilePath);
}
return _runtimeOptions;
}
public void SaveRunnerRuntimeOptions(RunnerRuntimeOptions options)
{
Trace.Info("Saving runtime options.");
if (File.Exists(_runtimeOptionsFilePath))
{
// Delete existing runtime options file first, since the file is hidden and not able to overwrite.
Trace.Info("Delete exist runtime options file.");
IOUtil.DeleteFile(_runtimeOptionsFilePath);
}
IOUtil.SaveObject(options, _runtimeOptionsFilePath);
Trace.Info("Options Saved.");
File.SetAttributes(_runtimeOptionsFilePath, File.GetAttributes(_runtimeOptionsFilePath) | FileAttributes.Hidden);
}
public void DeleteRunnerRuntimeOptions()
{
IOUtil.Delete(_runtimeOptionsFilePath, default(CancellationToken));
}
}
}

View File

@@ -0,0 +1,343 @@
using System;
namespace GitHub.Runner.Common
{
public enum RunMode
{
Normal, // Keep "Normal" first (default value).
Local,
}
public enum WellKnownDirectory
{
Bin,
Diag,
Externals,
Root,
Actions,
Temp,
Tools,
Update,
Work,
}
public enum WellKnownConfigFile
{
Runner,
Credentials,
RSACredentials,
Service,
CredentialStore,
Certificates,
Proxy,
ProxyCredentials,
ProxyBypass,
Options,
}
public static class Constants
{
/// <summary>Path environment variable name.</summary>
#if OS_WINDOWS
public static readonly string PathVariable = "Path";
#else
public static readonly string PathVariable = "PATH";
#endif
public static string ProcessTrackingId = "RUNNER_TRACKING_ID";
public static string PluginTracePrefix = "##[plugin.trace]";
public static readonly int RunnerDownloadRetryMaxAttempts = 3;
// This enum is embedded within the Constants class to make it easier to reference and avoid
// ambiguous type reference with System.Runtime.InteropServices.OSPlatform and System.Runtime.InteropServices.Architecture
public enum OSPlatform
{
OSX,
Linux,
Windows
}
public enum Architecture
{
X86,
X64,
Arm,
Arm64
}
public static class Runner
{
#if OS_LINUX
public static readonly OSPlatform Platform = OSPlatform.Linux;
#elif OS_OSX
public static readonly OSPlatform Platform = OSPlatform.OSX;
#elif OS_WINDOWS
public static readonly OSPlatform Platform = OSPlatform.Windows;
#endif
#if X86
public static readonly Architecture PlatformArchitecture = Architecture.X86;
#elif X64
public static readonly Architecture PlatformArchitecture = Architecture.X64;
#elif ARM
public static readonly Architecture PlatformArchitecture = Architecture.Arm;
#elif ARM64
public static readonly Architecture PlatformArchitecture = Architecture.Arm64;
#endif
public static readonly TimeSpan ExitOnUnloadTimeout = TimeSpan.FromSeconds(30);
public static class CommandLine
{
//if you are adding a new arg, please make sure you update the
//validArgs array as well present in the CommandSettings.cs
public static class Args
{
public static readonly string Agent = "agent";
public static readonly string Auth = "auth";
public static readonly string CollectionName = "collectionname";
public static readonly string DeploymentGroupName = "deploymentgroupname";
public static readonly string DeploymentPoolName = "deploymentpoolname";
public static readonly string DeploymentGroupTags = "deploymentgrouptags";
public static readonly string MachineGroupName = "machinegroupname";
public static readonly string MachineGroupTags = "machinegrouptags";
public static readonly string Matrix = "matrix";
public static readonly string MonitorSocketAddress = "monitorsocketaddress";
public static readonly string NotificationPipeName = "notificationpipename";
public static readonly string NotificationSocketAddress = "notificationsocketaddress";
public static readonly string Pool = "pool";
public static readonly string ProjectName = "projectname";
public static readonly string ProxyUrl = "proxyurl";
public static readonly string ProxyUserName = "proxyusername";
public static readonly string SslCACert = "sslcacert";
public static readonly string SslClientCert = "sslclientcert";
public static readonly string SslClientCertKey = "sslclientcertkey";
public static readonly string SslClientCertArchive = "sslclientcertarchive";
public static readonly string SslClientCertPassword = "sslclientcertpassword";
public static readonly string StartupType = "startuptype";
public static readonly string Url = "url";
public static readonly string UserName = "username";
public static readonly string WindowsLogonAccount = "windowslogonaccount";
public static readonly string Work = "work";
public static readonly string Yml = "yml";
// Secret args. Must be added to the "Secrets" getter as well.
public static readonly string Password = "password";
public static readonly string ProxyPassword = "proxypassword";
public static readonly string Token = "token";
public static readonly string WindowsLogonPassword = "windowslogonpassword";
public static string[] Secrets => new[]
{
Password,
ProxyPassword,
SslClientCertPassword,
Token,
WindowsLogonPassword,
};
}
public static class Commands
{
public static readonly string Configure = "configure";
public static readonly string LocalRun = "localRun";
public static readonly string Remove = "remove";
public static readonly string Run = "run";
public static readonly string Warmup = "warmup";
}
//if you are adding a new flag, please make sure you update the
//validFlags array as well present in the CommandSettings.cs
public static class Flags
{
public static readonly string AcceptTeeEula = "acceptteeeula";
public static readonly string AddDeploymentGroupTags = "adddeploymentgrouptags";
public static readonly string AddMachineGroupTags = "addmachinegrouptags";
public static readonly string Commit = "commit";
public static readonly string DeploymentGroup = "deploymentgroup";
public static readonly string DeploymentPool = "deploymentpool";
public static readonly string OverwriteAutoLogon = "overwriteautologon";
public static readonly string GitUseSChannel = "gituseschannel";
public static readonly string Help = "help";
public static readonly string MachineGroup = "machinegroup";
public static readonly string Replace = "replace";
public static readonly string NoRestart = "norestart";
public static readonly string LaunchBrowser = "launchbrowser";
public static readonly string Once = "once";
public static readonly string RunAsAutoLogon = "runasautologon";
public static readonly string RunAsService = "runasservice";
public static readonly string SslSkipCertValidation = "sslskipcertvalidation";
public static readonly string Unattended = "unattended";
public static readonly string Version = "version";
public static readonly string WhatIf = "whatif";
}
}
public static class ReturnCode
{
public const int Success = 0;
public const int TerminatedError = 1;
public const int RetryableError = 2;
public const int RunnerUpdating = 3;
public const int RunOnceRunnerUpdating = 4;
}
}
public static class Pipeline
{
public static class Path
{
public static readonly string PipelineMappingDirectory = "_PipelineMapping";
public static readonly string TrackingConfigFile = "PipelineFolder.json";
}
}
public static class Configuration
{
public static readonly string AAD = "AAD";
public static readonly string OAuthAccessToken = "OAuthAccessToken";
public static readonly string PAT = "PAT";
public static readonly string OAuth = "OAuth";
}
public static class Expressions
{
public static readonly string Always = "always";
public static readonly string Canceled = "canceled";
public static readonly string Cancelled = "cancelled";
public static readonly string Failed = "failed";
public static readonly string Failure = "failure";
public static readonly string Success = "success";
public static readonly string Succeeded = "succeeded";
public static readonly string SucceededOrFailed = "succeededOrFailed";
public static readonly string Variables = "variables";
}
public static class Path
{
public static readonly string ActionsDirectory = "_actions";
public static readonly string ActionManifestFile = "action.yml";
public static readonly string BinDirectory = "bin";
public static readonly string DiagDirectory = "_diag";
public static readonly string ExternalsDirectory = "externals";
public static readonly string RunnerDiagnosticLogPrefix = "Runner_";
public static readonly string TempDirectory = "_temp";
public static readonly string TeeDirectory = "tee";
public static readonly string ToolDirectory = "_tool";
public static readonly string TaskJsonFile = "task.json";
public static readonly string UpdateDirectory = "_update";
public static readonly string WorkDirectory = "_work";
public static readonly string WorkerDiagnosticLogPrefix = "Worker_";
}
// Related to definition variables.
public static class Variables
{
public static readonly string MacroPrefix = "$(";
public static readonly string MacroSuffix = ")";
public static class Actions
{
//
// Keep alphabetical
//
public static readonly string RunnerDebug = "ACTIONS_RUNNER_DEBUG";
public static readonly string StepDebug = "ACTIONS_STEP_DEBUG";
}
public static class Agent
{
//
// Keep alphabetical
//
public static readonly string AcceptTeeEula = "agent.acceptteeeula";
public static readonly string AllowAllEndpoints = "agent.allowAllEndpoints"; // remove after sprint 120 or so.
public static readonly string AllowAllSecureFiles = "agent.allowAllSecureFiles"; // remove after sprint 121 or so.
public static readonly string BuildDirectory = "agent.builddirectory";
public static readonly string ContainerId = "agent.containerid";
public static readonly string ContainerNetwork = "agent.containernetwork";
public static readonly string HomeDirectory = "agent.homedirectory";
public static readonly string Id = "agent.id";
public static readonly string GitUseSChannel = "agent.gituseschannel";
public static readonly string JobName = "agent.jobname";
public static readonly string MachineName = "agent.machinename";
public static readonly string Name = "agent.name";
public static readonly string OS = "agent.os";
public static readonly string OSArchitecture = "agent.osarchitecture";
public static readonly string OSVersion = "agent.osversion";
public static readonly string ProxyUrl = "agent.proxyurl";
public static readonly string ProxyUsername = "agent.proxyusername";
public static readonly string ProxyPassword = "agent.proxypassword";
public static readonly string ProxyBypassList = "agent.proxybypasslist";
public static readonly string RetainDefaultEncoding = "agent.retainDefaultEncoding";
public static readonly string RootDirectory = "agent.RootDirectory";
public static readonly string RunMode = "agent.runmode";
public static readonly string ServerOMDirectory = "agent.ServerOMDirectory";
public static readonly string ServicePortPrefix = "agent.services";
public static readonly string SslCAInfo = "agent.cainfo";
public static readonly string SslClientCert = "agent.clientcert";
public static readonly string SslClientCertKey = "agent.clientcertkey";
public static readonly string SslClientCertArchive = "agent.clientcertarchive";
public static readonly string SslClientCertPassword = "agent.clientcertpassword";
public static readonly string SslSkipCertValidation = "agent.skipcertvalidation";
public static readonly string TempDirectory = "agent.TempDirectory";
public static readonly string ToolsDirectory = "agent.ToolsDirectory";
public static readonly string Version = "agent.version";
public static readonly string WorkFolder = "agent.workfolder";
public static readonly string WorkingDirectory = "agent.WorkingDirectory";
}
public static class Build
{
//
// Keep alphabetical
//
public static readonly string ArtifactStagingDirectory = "build.artifactstagingdirectory";
public static readonly string BinariesDirectory = "build.binariesdirectory";
public static readonly string Number = "build.buildNumber";
public static readonly string Clean = "build.clean";
public static readonly string DefinitionName = "build.definitionname";
public static readonly string GatedRunCI = "build.gated.runci";
public static readonly string GatedShelvesetName = "build.gated.shelvesetname";
public static readonly string RepoClean = "build.repository.clean";
public static readonly string RepoGitSubmoduleCheckout = "build.repository.git.submodulecheckout";
public static readonly string RepoId = "build.repository.id";
public static readonly string RepoLocalPath = "build.repository.localpath";
public static readonly string RepoName = "build.Repository.name";
public static readonly string RepoProvider = "build.repository.provider";
public static readonly string RepoTfvcWorkspace = "build.repository.tfvc.workspace";
public static readonly string RepoUri = "build.repository.uri";
public static readonly string SourceBranch = "build.sourcebranch";
public static readonly string SourceTfvcShelveset = "build.sourcetfvcshelveset";
public static readonly string SourceVersion = "build.sourceversion";
public static readonly string SourcesDirectory = "build.sourcesdirectory";
public static readonly string StagingDirectory = "build.stagingdirectory";
public static readonly string SyncSources = "build.syncSources";
}
public static class System
{
//
// Keep alphabetical
//
public static readonly string AccessToken = "system.accessToken";
public static readonly string ArtifactsDirectory = "system.artifactsdirectory";
public static readonly string CollectionId = "system.collectionid";
public static readonly string Culture = "system.culture";
public static readonly string DefaultWorkingDirectory = "system.defaultworkingdirectory";
public static readonly string DefinitionId = "system.definitionid";
public static readonly string EnableAccessToken = "system.enableAccessToken";
public static readonly string HostType = "system.hosttype";
public static readonly string PhaseDisplayName = "system.phaseDisplayName";
public static readonly string PreferGitFromPath = "system.prefergitfrompath";
public static readonly string PullRequestTargetBranchName = "system.pullrequest.targetbranch";
public static readonly string SelfManageGitCreds = "system.selfmanagegitcreds";
public static readonly string ServerType = "system.servertype";
public static readonly string TFServerUrl = "system.TeamFoundationServerUri"; // back compat variable, do not document
public static readonly string TeamProject = "system.teamproject";
public static readonly string TeamProjectId = "system.teamProjectId";
public static readonly string WorkFolder = "system.workfolder";
}
}
}
}

View File

@@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
namespace GitHub.Runner.Common
{
public sealed class CredentialData
{
public string Scheme { get; set; }
public Dictionary<string, string> Data
{
get
{
if (_data == null)
{
_data = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
}
return _data;
}
}
private Dictionary<string, string> _data;
}
}

View File

@@ -0,0 +1,19 @@
using System;
namespace GitHub.Runner.Common
{
public class NonRetryableException : Exception
{
public NonRetryableException()
: base()
{ }
public NonRetryableException(string message)
: base(message)
{ }
public NonRetryableException(string message, Exception inner)
: base(message, inner)
{ }
}
}

View File

@@ -0,0 +1,80 @@
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
namespace GitHub.Runner.Common
{
[ServiceLocator(Default = typeof(ExtensionManager))]
public interface IExtensionManager : IRunnerService
{
List<T> GetExtensions<T>() where T : class, IExtension;
}
public sealed class ExtensionManager : RunnerService, IExtensionManager
{
private readonly ConcurrentDictionary<Type, List<IExtension>> _cache = new ConcurrentDictionary<Type, List<IExtension>>();
public List<T> GetExtensions<T>() where T : class, IExtension
{
Trace.Info("Getting extensions for interface: '{0}'", typeof(T).FullName);
List<IExtension> extensions = _cache.GetOrAdd(
key: typeof(T),
valueFactory: (Type key) =>
{
return LoadExtensions<T>();
});
return extensions.Select(x => x as T).ToList();
}
//
// We will load extensions from assembly
// once AssemblyLoadContext.Resolving event is able to
// resolve dependency recursively
//
private List<IExtension> LoadExtensions<T>() where T : class, IExtension
{
var extensions = new List<IExtension>();
switch (typeof(T).FullName)
{
// Listener capabilities providers.
case "GitHub.Runner.Common.Capabilities.ICapabilitiesProvider":
Add<T>(extensions, "GitHub.Runner.Common.Capabilities.RunnerCapabilitiesProvider, Runner.Common");
break;
// Action command extensions.
case "GitHub.Runner.Worker.IActionCommandExtension":
Add<T>(extensions, "GitHub.Runner.Worker.InternalPluginSetRepoPathCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.SetEnvCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.SetOutputCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.SaveStateCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.AddPathCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.AddMaskCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.AddMatcherCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.RemoveMatcherCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.WarningCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.ErrorCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.DebugCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.GroupCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.EndGroupCommandExtension, Runner.Worker");
break;
default:
// This should never happen.
throw new NotSupportedException($"Unexpected extension type: '{typeof(T).FullName}'");
}
return extensions;
}
private void Add<T>(List<IExtension> extensions, string assemblyQualifiedName) where T : class, IExtension
{
Trace.Info($"Creating instance: {assemblyQualifiedName}");
Type type = Type.GetType(assemblyQualifiedName, throwOnError: true);
var extension = Activator.CreateInstance(type) as T;
ArgUtil.NotNull(extension, nameof(extension));
extension.Initialize(HostContext);
extensions.Add(extension);
}
}
}

View File

@@ -0,0 +1,30 @@
using System;
using System.Threading;
using System.Threading.Tasks;
namespace GitHub.Runner.Common
{
//this code is documented on http://blogs.msdn.com/b/pfxteam/archive/2012/10/05/how-do-i-cancel-non-cancelable-async-operations.aspx
public static class Extensions
{
public static async Task<T> WithCancellation<T>(this Task<T> task, CancellationToken cancellationToken)
{
var tcs = new TaskCompletionSource<bool>();
using (cancellationToken.Register(
s => ((TaskCompletionSource<bool>)s).TrySetResult(true), tcs))
if (task != await Task.WhenAny(task, tcs.Task))
throw new OperationCanceledException(cancellationToken);
return await task;
}
public static async Task WithCancellation(this Task task, CancellationToken cancellationToken)
{
var tcs = new TaskCompletionSource<bool>();
using (cancellationToken.Register(
s => ((TaskCompletionSource<bool>)s).TrySetResult(true), tcs))
if (task != await Task.WhenAny(task, tcs.Task))
throw new OperationCanceledException(cancellationToken);
await task;
}
}
}

View File

@@ -0,0 +1,597 @@
using GitHub.Runner.Common.Util;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.Loader;
using System.Threading;
using System.Threading.Tasks;
using System.Diagnostics;
using System.Net.Http;
using System.Diagnostics.Tracing;
using GitHub.DistributedTask.Logging;
using System.Net.Http.Headers;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common
{
public interface IHostContext : IDisposable
{
RunMode RunMode { get; set; }
StartupType StartupType { get; set; }
CancellationToken RunnerShutdownToken { get; }
ShutdownReason RunnerShutdownReason { get; }
ISecretMasker SecretMasker { get; }
ProductInfoHeaderValue UserAgent { get; }
string GetDirectory(WellKnownDirectory directory);
string GetConfigFile(WellKnownConfigFile configFile);
Tracing GetTrace(string name);
Task Delay(TimeSpan delay, CancellationToken cancellationToken);
T CreateService<T>() where T : class, IRunnerService;
T GetService<T>() where T : class, IRunnerService;
void SetDefaultCulture(string name);
event EventHandler Unloading;
void ShutdownRunner(ShutdownReason reason);
void WritePerfCounter(string counter);
}
public enum StartupType
{
Manual,
Service,
AutoStartup
}
public sealed class HostContext : EventListener, IObserver<DiagnosticListener>, IObserver<KeyValuePair<string, object>>, IHostContext, IDisposable
{
private const int _defaultLogPageSize = 8; //MB
private static int _defaultLogRetentionDays = 30;
private static int[] _vssHttpMethodEventIds = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 24 };
private static int[] _vssHttpCredentialEventIds = new int[] { 11, 13, 14, 15, 16, 17, 18, 20, 21, 22, 27, 29 };
private readonly ConcurrentDictionary<Type, object> _serviceInstances = new ConcurrentDictionary<Type, object>();
private readonly ConcurrentDictionary<Type, Type> _serviceTypes = new ConcurrentDictionary<Type, Type>();
private readonly ISecretMasker _secretMasker = new SecretMasker();
private readonly ProductInfoHeaderValue _userAgent = new ProductInfoHeaderValue($"GitHubActionsRunner-{BuildConstants.RunnerPackage.PackageName}", BuildConstants.RunnerPackage.Version);
private CancellationTokenSource _runnerShutdownTokenSource = new CancellationTokenSource();
private object _perfLock = new object();
private RunMode _runMode = RunMode.Normal;
private Tracing _trace;
private Tracing _vssTrace;
private Tracing _httpTrace;
private ITraceManager _traceManager;
private AssemblyLoadContext _loadContext;
private IDisposable _httpTraceSubscription;
private IDisposable _diagListenerSubscription;
private StartupType _startupType;
private string _perfFile;
public event EventHandler Unloading;
public CancellationToken RunnerShutdownToken => _runnerShutdownTokenSource.Token;
public ShutdownReason RunnerShutdownReason { get; private set; }
public ISecretMasker SecretMasker => _secretMasker;
public ProductInfoHeaderValue UserAgent => _userAgent;
public HostContext(string hostType, string logFile = null)
{
// Validate args.
ArgUtil.NotNullOrEmpty(hostType, nameof(hostType));
_loadContext = AssemblyLoadContext.GetLoadContext(typeof(HostContext).GetTypeInfo().Assembly);
_loadContext.Unloading += LoadContext_Unloading;
this.SecretMasker.AddValueEncoder(ValueEncoders.Base64StringEscape);
this.SecretMasker.AddValueEncoder(ValueEncoders.Base64StringEscapeShift1);
this.SecretMasker.AddValueEncoder(ValueEncoders.Base64StringEscapeShift2);
this.SecretMasker.AddValueEncoder(ValueEncoders.Base64StringEscapeShift3);
this.SecretMasker.AddValueEncoder(ValueEncoders.Base64StringEscapeShift4);
this.SecretMasker.AddValueEncoder(ValueEncoders.Base64StringEscapeShift5);
this.SecretMasker.AddValueEncoder(ValueEncoders.ExpressionStringEscape);
this.SecretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape);
this.SecretMasker.AddValueEncoder(ValueEncoders.UriDataEscape);
this.SecretMasker.AddValueEncoder(ValueEncoders.XmlDataEscape);
// Create the trace manager.
if (string.IsNullOrEmpty(logFile))
{
int logPageSize;
string logSizeEnv = Environment.GetEnvironmentVariable($"{hostType.ToUpperInvariant()}_LOGSIZE");
if (!string.IsNullOrEmpty(logSizeEnv) || !int.TryParse(logSizeEnv, out logPageSize))
{
logPageSize = _defaultLogPageSize;
}
int logRetentionDays;
string logRetentionDaysEnv = Environment.GetEnvironmentVariable($"{hostType.ToUpperInvariant()}_LOGRETENTION");
if (!string.IsNullOrEmpty(logRetentionDaysEnv) || !int.TryParse(logRetentionDaysEnv, out logRetentionDays))
{
logRetentionDays = _defaultLogRetentionDays;
}
// this should give us _diag folder under runner root directory
string diagLogDirectory = Path.Combine(new DirectoryInfo(Path.GetDirectoryName(Assembly.GetEntryAssembly().Location)).Parent.FullName, Constants.Path.DiagDirectory);
_traceManager = new TraceManager(new HostTraceListener(diagLogDirectory, hostType, logPageSize, logRetentionDays), this.SecretMasker);
}
else
{
_traceManager = new TraceManager(new HostTraceListener(logFile), this.SecretMasker);
}
_trace = GetTrace(nameof(HostContext));
_vssTrace = GetTrace("GitHubActionsRunner"); // VisualStudioService
// Enable Http trace
bool enableHttpTrace;
if (bool.TryParse(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_HTTPTRACE"), out enableHttpTrace) && enableHttpTrace)
{
_trace.Warning("*****************************************************************************************");
_trace.Warning("** **");
_trace.Warning("** Http trace is enabled, all your http traffic will be dumped into runner diag log. **");
_trace.Warning("** DO NOT share the log in public place! The trace may contains secrets in plain text. **");
_trace.Warning("** **");
_trace.Warning("*****************************************************************************************");
_httpTrace = GetTrace("HttpTrace");
_diagListenerSubscription = DiagnosticListener.AllListeners.Subscribe(this);
}
// Enable perf counter trace
string perfCounterLocation = Environment.GetEnvironmentVariable("RUNNER_PERFLOG");
if (!string.IsNullOrEmpty(perfCounterLocation))
{
try
{
Directory.CreateDirectory(perfCounterLocation);
_perfFile = Path.Combine(perfCounterLocation, $"{hostType}.perf");
}
catch (Exception ex)
{
_trace.Error(ex);
}
}
}
public RunMode RunMode
{
get
{
return _runMode;
}
set
{
_trace.Info($"Set run mode: {value}");
_runMode = value;
}
}
public string GetDirectory(WellKnownDirectory directory)
{
string path;
switch (directory)
{
case WellKnownDirectory.Bin:
path = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
break;
case WellKnownDirectory.Diag:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
Constants.Path.DiagDirectory);
break;
case WellKnownDirectory.Externals:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
Constants.Path.ExternalsDirectory);
break;
case WellKnownDirectory.Root:
path = new DirectoryInfo(GetDirectory(WellKnownDirectory.Bin)).Parent.FullName;
break;
case WellKnownDirectory.Temp:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Work),
Constants.Path.TempDirectory);
break;
case WellKnownDirectory.Actions:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Work),
Constants.Path.ActionsDirectory);
break;
case WellKnownDirectory.Tools:
// TODO: Coallesce to just check RUNNER_TOOL_CACHE when images stabilize
path = Environment.GetEnvironmentVariable("RUNNER_TOOL_CACHE") ?? Environment.GetEnvironmentVariable("RUNNER_TOOLSDIRECTORY") ?? Environment.GetEnvironmentVariable("AGENT_TOOLSDIRECTORY") ?? Environment.GetEnvironmentVariable(Constants.Variables.Agent.ToolsDirectory);
if (string.IsNullOrEmpty(path))
{
path = Path.Combine(
GetDirectory(WellKnownDirectory.Work),
Constants.Path.ToolDirectory);
}
break;
case WellKnownDirectory.Update:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Work),
Constants.Path.UpdateDirectory);
break;
case WellKnownDirectory.Work:
var configurationStore = GetService<IConfigurationStore>();
RunnerSettings settings = configurationStore.GetSettings();
ArgUtil.NotNull(settings, nameof(settings));
ArgUtil.NotNullOrEmpty(settings.WorkFolder, nameof(settings.WorkFolder));
path = Path.GetFullPath(Path.Combine(
GetDirectory(WellKnownDirectory.Root),
settings.WorkFolder));
break;
default:
throw new NotSupportedException($"Unexpected well known directory: '{directory}'");
}
_trace.Info($"Well known directory '{directory}': '{path}'");
return path;
}
public string GetConfigFile(WellKnownConfigFile configFile)
{
string path;
switch (configFile)
{
case WellKnownConfigFile.Runner:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
".runner");
break;
case WellKnownConfigFile.Credentials:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
".credentials");
break;
case WellKnownConfigFile.RSACredentials:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
".credentials_rsaparams");
break;
case WellKnownConfigFile.Service:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
".service");
break;
case WellKnownConfigFile.CredentialStore:
#if OS_OSX
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
".credential_store.keychain");
#else
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
".credential_store");
#endif
break;
case WellKnownConfigFile.Certificates:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
".certificates");
break;
case WellKnownConfigFile.Proxy:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
".proxy");
break;
case WellKnownConfigFile.ProxyCredentials:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
".proxycredentials");
break;
case WellKnownConfigFile.ProxyBypass:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
".proxybypass");
break;
case WellKnownConfigFile.Options:
path = Path.Combine(
GetDirectory(WellKnownDirectory.Root),
".options");
break;
default:
throw new NotSupportedException($"Unexpected well known config file: '{configFile}'");
}
_trace.Info($"Well known config file '{configFile}': '{path}'");
return path;
}
public Tracing GetTrace(string name)
{
return _traceManager[name];
}
public async Task Delay(TimeSpan delay, CancellationToken cancellationToken)
{
await Task.Delay(delay, cancellationToken);
}
/// <summary>
/// Creates a new instance of T.
/// </summary>
public T CreateService<T>() where T : class, IRunnerService
{
Type target;
if (!_serviceTypes.TryGetValue(typeof(T), out target))
{
// Infer the concrete type from the ServiceLocatorAttribute.
CustomAttributeData attribute = typeof(T)
.GetTypeInfo()
.CustomAttributes
.FirstOrDefault(x => x.AttributeType == typeof(ServiceLocatorAttribute));
if (attribute != null)
{
foreach (CustomAttributeNamedArgument arg in attribute.NamedArguments)
{
if (string.Equals(arg.MemberName, ServiceLocatorAttribute.DefaultPropertyName, StringComparison.Ordinal))
{
target = arg.TypedValue.Value as Type;
}
}
}
if (target == null)
{
throw new KeyNotFoundException(string.Format(CultureInfo.InvariantCulture, "Service mapping not found for key '{0}'.", typeof(T).FullName));
}
_serviceTypes.TryAdd(typeof(T), target);
target = _serviceTypes[typeof(T)];
}
// Create a new instance.
T svc = Activator.CreateInstance(target) as T;
svc.Initialize(this);
return svc;
}
/// <summary>
/// Gets or creates an instance of T.
/// </summary>
public T GetService<T>() where T : class, IRunnerService
{
// Return the cached instance if one already exists.
object instance;
if (_serviceInstances.TryGetValue(typeof(T), out instance))
{
return instance as T;
}
// Otherwise create a new instance and try to add it to the cache.
_serviceInstances.TryAdd(typeof(T), CreateService<T>());
// Return the instance from the cache.
return _serviceInstances[typeof(T)] as T;
}
public void SetDefaultCulture(string name)
{
ArgUtil.NotNull(name, nameof(name));
_trace.Verbose($"Setting default culture and UI culture to: '{name}'");
CultureInfo.DefaultThreadCurrentCulture = new CultureInfo(name);
CultureInfo.DefaultThreadCurrentUICulture = new CultureInfo(name);
}
public void ShutdownRunner(ShutdownReason reason)
{
ArgUtil.NotNull(reason, nameof(reason));
_trace.Info($"Runner will be shutdown for {reason.ToString()}");
RunnerShutdownReason = reason;
_runnerShutdownTokenSource.Cancel();
}
public override void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
public StartupType StartupType
{
get
{
return _startupType;
}
set
{
_startupType = value;
}
}
public void WritePerfCounter(string counter)
{
if (!string.IsNullOrEmpty(_perfFile))
{
string normalizedCounter = counter.Replace(':', '_');
lock (_perfLock)
{
try
{
File.AppendAllLines(_perfFile, new[] { $"{normalizedCounter}:{DateTime.UtcNow.ToString("O")}" });
}
catch (Exception ex)
{
_trace.Error(ex);
}
}
}
}
private void Dispose(bool disposing)
{
// TODO: Dispose the trace listener also.
if (disposing)
{
if (_loadContext != null)
{
_loadContext.Unloading -= LoadContext_Unloading;
_loadContext = null;
}
_httpTraceSubscription?.Dispose();
_diagListenerSubscription?.Dispose();
_traceManager?.Dispose();
_traceManager = null;
_runnerShutdownTokenSource?.Dispose();
_runnerShutdownTokenSource = null;
base.Dispose();
}
}
private void LoadContext_Unloading(AssemblyLoadContext obj)
{
if (Unloading != null)
{
Unloading(this, null);
}
}
void IObserver<DiagnosticListener>.OnCompleted()
{
_httpTrace.Info("DiagListeners finished transmitting data.");
}
void IObserver<DiagnosticListener>.OnError(Exception error)
{
_httpTrace.Error(error);
}
void IObserver<DiagnosticListener>.OnNext(DiagnosticListener listener)
{
if (listener.Name == "HttpHandlerDiagnosticListener" && _httpTraceSubscription == null)
{
_httpTraceSubscription = listener.Subscribe(this);
}
}
void IObserver<KeyValuePair<string, object>>.OnCompleted()
{
_httpTrace.Info("HttpHandlerDiagnosticListener finished transmitting data.");
}
void IObserver<KeyValuePair<string, object>>.OnError(Exception error)
{
_httpTrace.Error(error);
}
void IObserver<KeyValuePair<string, object>>.OnNext(KeyValuePair<string, object> value)
{
_httpTrace.Info($"Trace {value.Key} event:{Environment.NewLine}{value.Value.ToString()}");
}
protected override void OnEventSourceCreated(EventSource source)
{
if (source.Name.Equals("Microsoft-VSS-Http"))
{
EnableEvents(source, EventLevel.Verbose);
}
}
protected override void OnEventWritten(EventWrittenEventArgs eventData)
{
if (eventData == null)
{
return;
}
string message = eventData.Message;
object[] payload = new object[0];
if (eventData.Payload != null && eventData.Payload.Count > 0)
{
payload = eventData.Payload.ToArray();
}
try
{
if (_vssHttpMethodEventIds.Contains(eventData.EventId))
{
payload[0] = Enum.Parse(typeof(VssHttpMethod), ((int)payload[0]).ToString());
}
else if (_vssHttpCredentialEventIds.Contains(eventData.EventId))
{
payload[0] = Enum.Parse(typeof(GitHub.Services.Common.VssCredentialsType), ((int)payload[0]).ToString());
}
if (payload.Length > 0)
{
message = String.Format(eventData.Message.Replace("%n", Environment.NewLine), payload);
}
switch (eventData.Level)
{
case EventLevel.Critical:
case EventLevel.Error:
_vssTrace.Error(message);
break;
case EventLevel.Warning:
_vssTrace.Warning(message);
break;
case EventLevel.Informational:
_vssTrace.Info(message);
break;
default:
_vssTrace.Verbose(message);
break;
}
}
catch (Exception ex)
{
_vssTrace.Error(ex);
_vssTrace.Info(eventData.Message);
_vssTrace.Info(string.Join(", ", eventData.Payload?.ToArray() ?? new string[0]));
}
}
// Copied from pipelines server code base, used for EventData translation.
internal enum VssHttpMethod
{
UNKNOWN,
DELETE,
HEAD,
GET,
OPTIONS,
PATCH,
POST,
PUT,
}
}
public static class HostContextExtension
{
public static HttpClientHandler CreateHttpClientHandler(this IHostContext context)
{
HttpClientHandler clientHandler = new HttpClientHandler();
var runnerWebProxy = context.GetService<IRunnerWebProxy>();
clientHandler.Proxy = runnerWebProxy.WebProxy;
return clientHandler;
}
}
public enum ShutdownReason
{
UserCancelled = 0,
OperatingSystemShutdown = 1,
}
}

View File

@@ -0,0 +1,202 @@
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using System;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Text;
namespace GitHub.Runner.Common
{
public sealed class HostTraceListener : TextWriterTraceListener
{
private const string _logFileNamingPattern = "{0}_{1:yyyyMMdd-HHmmss}-utc.log";
private string _logFileDirectory;
private string _logFilePrefix;
private bool _enablePageLog = false;
private bool _enableLogRetention = false;
private int _currentPageSize;
private int _pageSizeLimit;
private int _retentionDays;
public HostTraceListener(string logFileDirectory, string logFilePrefix, int pageSizeLimit, int retentionDays)
: base()
{
ArgUtil.NotNullOrEmpty(logFileDirectory, nameof(logFileDirectory));
ArgUtil.NotNullOrEmpty(logFilePrefix, nameof(logFilePrefix));
_logFileDirectory = logFileDirectory;
_logFilePrefix = logFilePrefix;
Directory.CreateDirectory(_logFileDirectory);
if (pageSizeLimit > 0)
{
_enablePageLog = true;
_pageSizeLimit = pageSizeLimit * 1024 * 1024;
_currentPageSize = 0;
}
if (retentionDays > 0)
{
_enableLogRetention = true;
_retentionDays = retentionDays;
}
Writer = CreatePageLogWriter();
}
public HostTraceListener(string logFile)
: base()
{
ArgUtil.NotNullOrEmpty(logFile, nameof(logFile));
Directory.CreateDirectory(Path.GetDirectoryName(logFile));
Stream logStream = new FileStream(logFile, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, bufferSize: 4096);
Writer = new StreamWriter(logStream);
}
// Copied and modified slightly from .Net Core source code. Modification was required to make it compile.
// There must be some TraceFilter extension class that is missing in this source code.
public override void TraceEvent(TraceEventCache eventCache, string source, TraceEventType eventType, int id, string message)
{
if (Filter != null && !Filter.ShouldTrace(eventCache, source, eventType, id, message, null, null, null))
{
return;
}
WriteHeader(source, eventType, id);
WriteLine(message);
WriteFooter(eventCache);
}
public override void WriteLine(string message)
{
base.WriteLine(message);
if (_enablePageLog)
{
int messageSize = UTF8Encoding.UTF8.GetByteCount(message);
_currentPageSize += messageSize;
if (_currentPageSize > _pageSizeLimit)
{
Flush();
if (Writer != null)
{
Writer.Dispose();
Writer = null;
}
Writer = CreatePageLogWriter();
_currentPageSize = 0;
}
}
Flush();
}
public override void Write(string message)
{
base.Write(message);
if (_enablePageLog)
{
int messageSize = UTF8Encoding.UTF8.GetByteCount(message);
_currentPageSize += messageSize;
}
Flush();
}
internal bool IsEnabled(TraceOptions opts)
{
return (opts & TraceOutputOptions) != 0;
}
// Altered from the original .Net Core implementation.
private void WriteHeader(string source, TraceEventType eventType, int id)
{
string type = null;
switch (eventType)
{
case TraceEventType.Critical:
type = "CRIT";
break;
case TraceEventType.Error:
type = "ERR ";
break;
case TraceEventType.Warning:
type = "WARN";
break;
case TraceEventType.Information:
type = "INFO";
break;
case TraceEventType.Verbose:
type = "VERB";
break;
default:
type = eventType.ToString();
break;
}
Write(StringUtil.Format("[{0:u} {1} {2}] ", DateTime.UtcNow, type, source));
}
// Copied and modified slightly from .Net Core source code to make it compile. The original code
// accesses a private indentLevel field. In this code it has been modified to use the getter/setter.
private void WriteFooter(TraceEventCache eventCache)
{
if (eventCache == null)
return;
IndentLevel++;
if (IsEnabled(TraceOptions.ProcessId))
WriteLine("ProcessId=" + eventCache.ProcessId);
if (IsEnabled(TraceOptions.ThreadId))
WriteLine("ThreadId=" + eventCache.ThreadId);
if (IsEnabled(TraceOptions.DateTime))
WriteLine("DateTime=" + eventCache.DateTime.ToString("o", CultureInfo.InvariantCulture));
if (IsEnabled(TraceOptions.Timestamp))
WriteLine("Timestamp=" + eventCache.Timestamp);
IndentLevel--;
}
private StreamWriter CreatePageLogWriter()
{
if (_enableLogRetention)
{
DirectoryInfo diags = new DirectoryInfo(_logFileDirectory);
var logs = diags.GetFiles($"{_logFilePrefix}*.log");
foreach (var log in logs)
{
if (log.LastWriteTimeUtc.AddDays(_retentionDays) < DateTime.UtcNow)
{
try
{
log.Delete();
}
catch (Exception)
{
// catch Exception and continue
// we shouldn't block logging and fail the runner if the runner can't delete an older log file.
}
}
}
}
string fileName = StringUtil.Format(_logFileNamingPattern, _logFilePrefix, DateTime.UtcNow);
string logFile = Path.Combine(_logFileDirectory, fileName);
Stream logStream;
if (File.Exists(logFile))
{
logStream = new FileStream(logFile, FileMode.Append, FileAccess.Write, FileShare.Read, bufferSize: 4096);
}
else
{
logStream = new FileStream(logFile, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, bufferSize: 4096);
}
return new StreamWriter(logStream);
}
}
}

View File

@@ -0,0 +1,9 @@
using System;
namespace GitHub.Runner.Common
{
public interface IExtension : IRunnerService
{
Type ExtensionType { get; }
}
}

View File

@@ -0,0 +1,296 @@
using System;
using System.IO;
using System.IO.Pipes;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace GitHub.Runner.Common
{
[ServiceLocator(Default = typeof(JobNotification))]
public interface IJobNotification : IRunnerService, IDisposable
{
Task JobStarted(Guid jobId, string accessToken, Uri serverUrl);
Task JobCompleted(Guid jobId);
void StartClient(string pipeName, string monitorSocketAddress, CancellationToken cancellationToken);
void StartClient(string socketAddress, string monitorSocketAddress);
}
public sealed class JobNotification : RunnerService, IJobNotification
{
private NamedPipeClientStream _outClient;
private StreamWriter _writeStream;
private Socket _socket;
private Socket _monitorSocket;
private bool _configured = false;
private bool _useSockets = false;
private bool _isMonitorConfigured = false;
public async Task JobStarted(Guid jobId, string accessToken, Uri serverUrl)
{
Trace.Info("Entering JobStarted Notification");
StartMonitor(jobId, accessToken, serverUrl);
if (_configured)
{
String message = $"Starting job: {jobId.ToString()}";
if (_useSockets)
{
try
{
Trace.Info("Writing JobStarted to socket");
_socket.Send(Encoding.UTF8.GetBytes(message));
Trace.Info("Finished JobStarted writing to socket");
}
catch (SocketException e)
{
Trace.Error($"Failed sending message \"{message}\" on socket!");
Trace.Error(e);
}
}
else
{
Trace.Info("Writing JobStarted to pipe");
await _writeStream.WriteLineAsync(message);
await _writeStream.FlushAsync();
Trace.Info("Finished JobStarted writing to pipe");
}
}
}
public async Task JobCompleted(Guid jobId)
{
Trace.Info("Entering JobCompleted Notification");
await EndMonitor();
if (_configured)
{
String message = $"Finished job: {jobId.ToString()}";
if (_useSockets)
{
try
{
Trace.Info("Writing JobCompleted to socket");
_socket.Send(Encoding.UTF8.GetBytes(message));
Trace.Info("Finished JobCompleted writing to socket");
}
catch (SocketException e)
{
Trace.Error($"Failed sending message \"{message}\" on socket!");
Trace.Error(e);
}
}
else
{
Trace.Info("Writing JobCompleted to pipe");
await _writeStream.WriteLineAsync(message);
await _writeStream.FlushAsync();
Trace.Info("Finished JobCompleted writing to pipe");
}
}
}
public async void StartClient(string pipeName, string monitorSocketAddress, CancellationToken cancellationToken)
{
if (pipeName != null && !_configured)
{
Trace.Info("Connecting to named pipe {0}", pipeName);
_outClient = new NamedPipeClientStream(".", pipeName, PipeDirection.Out, PipeOptions.Asynchronous);
await _outClient.ConnectAsync(cancellationToken);
_writeStream = new StreamWriter(_outClient, Encoding.UTF8);
_configured = true;
Trace.Info("Connection successful to named pipe {0}", pipeName);
}
ConnectMonitor(monitorSocketAddress);
}
public void StartClient(string socketAddress, string monitorSocketAddress)
{
if (!_configured)
{
try
{
string[] splitAddress = socketAddress.Split(':');
if (splitAddress.Length != 2)
{
Trace.Error("Invalid socket address {0}. Job Notification will be disabled.", socketAddress);
return;
}
IPAddress address;
try
{
address = IPAddress.Parse(splitAddress[0]);
}
catch (FormatException e)
{
Trace.Error("Invalid socket ip address {0}. Job Notification will be disabled",splitAddress[0]);
Trace.Error(e);
return;
}
int port = -1;
Int32.TryParse(splitAddress[1], out port);
if (port < IPEndPoint.MinPort || port > IPEndPoint.MaxPort)
{
Trace.Error("Invalid tcp socket port {0}. Job Notification will be disabled.", splitAddress[1]);
return;
}
_socket = new Socket(SocketType.Stream, ProtocolType.Tcp);
_socket.Connect(address, port);
Trace.Info("Connection successful to socket {0}", socketAddress);
_useSockets = true;
_configured = true;
}
catch (SocketException e)
{
Trace.Error("Connection to socket {0} failed!", socketAddress);
Trace.Error(e);
}
}
ConnectMonitor(monitorSocketAddress);
}
private void StartMonitor(Guid jobId, string accessToken, Uri serverUri)
{
if(String.IsNullOrEmpty(accessToken))
{
Trace.Info("No access token could be retrieved to start the monitor.");
return;
}
try
{
Trace.Info("Entering StartMonitor");
if (_isMonitorConfigured)
{
String message = $"Start {jobId.ToString()} {accessToken} {serverUri.ToString()} {System.Diagnostics.Process.GetCurrentProcess().Id}";
Trace.Info("Writing StartMonitor to socket");
_monitorSocket.Send(Encoding.UTF8.GetBytes(message));
Trace.Info("Finished StartMonitor writing to socket");
}
}
catch (SocketException e)
{
Trace.Error($"Failed sending StartMonitor message on socket!");
Trace.Error(e);
}
catch (Exception e)
{
Trace.Error($"Unexpected error occurred while sending StartMonitor message on socket!");
Trace.Error(e);
}
}
private async Task EndMonitor()
{
try
{
Trace.Info("Entering EndMonitor");
if (_isMonitorConfigured)
{
String message = $"End {System.Diagnostics.Process.GetCurrentProcess().Id}";
Trace.Info("Writing EndMonitor to socket");
_monitorSocket.Send(Encoding.UTF8.GetBytes(message));
Trace.Info("Finished EndMonitor writing to socket");
await Task.Delay(TimeSpan.FromSeconds(2));
}
}
catch (SocketException e)
{
Trace.Error($"Failed sending end message on socket!");
Trace.Error(e);
}
catch (Exception e)
{
Trace.Error($"Unexpected error occurred while sending StartMonitor message on socket!");
Trace.Error(e);
}
}
private void ConnectMonitor(string monitorSocketAddress)
{
int port = -1;
if (!_isMonitorConfigured && !String.IsNullOrEmpty(monitorSocketAddress))
{
try
{
string[] splitAddress = monitorSocketAddress.Split(':');
if (splitAddress.Length != 2)
{
Trace.Error("Invalid socket address {0}. Unable to connect to monitor.", monitorSocketAddress);
return;
}
IPAddress address;
try
{
address = IPAddress.Parse(splitAddress[0]);
}
catch (FormatException e)
{
Trace.Error("Invalid socket IP address {0}. Unable to connect to monitor.", splitAddress[0]);
Trace.Error(e);
return;
}
Int32.TryParse(splitAddress[1], out port);
if (port < IPEndPoint.MinPort || port > IPEndPoint.MaxPort)
{
Trace.Error("Invalid TCP socket port {0}. Unable to connect to monitor.", splitAddress[1]);
return;
}
Trace.Verbose("Trying to connect to monitor at port {0}", port);
_monitorSocket = new Socket(SocketType.Stream, ProtocolType.Tcp);
_monitorSocket.Connect(address, port);
Trace.Info("Connection successful to local port {0}", port);
_isMonitorConfigured = true;
}
catch (Exception e)
{
Trace.Error("Connection to monitor port {0} failed!", port);
Trace.Error(e);
}
}
}
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
private void Dispose(bool disposing)
{
if (disposing)
{
_outClient?.Dispose();
if (_socket != null)
{
_socket.Send(Encoding.UTF8.GetBytes("<EOF>"));
_socket.Shutdown(SocketShutdown.Both);
_socket = null;
}
if (_monitorSocket != null)
{
_monitorSocket.Send(Encoding.UTF8.GetBytes("<EOF>"));
_monitorSocket.Shutdown(SocketShutdown.Both);
_monitorSocket = null;
}
}
}
}
}

View File

@@ -0,0 +1,162 @@
using GitHub.DistributedTask.WebApi;
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Services.WebApi;
namespace GitHub.Runner.Common
{
[ServiceLocator(Default = typeof(JobServer))]
public interface IJobServer : IRunnerService
{
Task ConnectAsync(VssConnection jobConnection);
// logging and console
Task<TaskLog> AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken);
Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, CancellationToken cancellationToken);
Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken);
Task<TaskLog> CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken);
Task<Timeline> CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken);
Task<List<TimelineRecord>> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable<TimelineRecord> records, CancellationToken cancellationToken);
Task RaisePlanEventAsync<T>(Guid scopeIdentifier, string hubName, Guid planId, T eventData, CancellationToken cancellationToken) where T : JobEvent;
Task<Timeline> GetTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken);
}
public sealed class JobServer : RunnerService, IJobServer
{
private bool _hasConnection;
private VssConnection _connection;
private TaskHttpClient _taskClient;
public async Task ConnectAsync(VssConnection jobConnection)
{
if (HostContext.RunMode == RunMode.Local)
{
return;
}
_connection = jobConnection;
int attemptCount = 5;
while (!_connection.HasAuthenticated && attemptCount-- > 0)
{
try
{
await _connection.ConnectAsync();
break;
}
catch (Exception ex) when (attemptCount > 0)
{
Trace.Info($"Catch exception during connect. {attemptCount} attemp left.");
Trace.Error(ex);
}
await Task.Delay(100);
}
_taskClient = _connection.GetClient<TaskHttpClient>();
_hasConnection = true;
}
private void CheckConnection()
{
if (!_hasConnection)
{
throw new InvalidOperationException("SetConnection");
}
}
//-----------------------------------------------------------------
// Feedback: WebConsole, TimelineRecords and Logs
//-----------------------------------------------------------------
public Task<TaskLog> AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken)
{
if (HostContext.RunMode == RunMode.Local)
{
return Task.FromResult<TaskLog>(null);
}
CheckConnection();
return _taskClient.AppendLogContentAsync(scopeIdentifier, hubName, planId, logId, uploadStream, cancellationToken: cancellationToken);
}
public Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, CancellationToken cancellationToken)
{
if (HostContext.RunMode == RunMode.Local)
{
return Task.CompletedTask;
}
CheckConnection();
return _taskClient.AppendTimelineRecordFeedAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, stepId, lines, cancellationToken: cancellationToken);
}
public Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, string type, string name, Stream uploadStream, CancellationToken cancellationToken)
{
if (HostContext.RunMode == RunMode.Local)
{
return Task.FromResult<TaskAttachment>(null);
}
CheckConnection();
return _taskClient.CreateAttachmentAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, type, name, uploadStream, cancellationToken: cancellationToken);
}
public Task<TaskLog> CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken)
{
if (HostContext.RunMode == RunMode.Local)
{
return Task.FromResult<TaskLog>(null);
}
CheckConnection();
return _taskClient.CreateLogAsync(scopeIdentifier, hubName, planId, log, cancellationToken: cancellationToken);
}
public Task<Timeline> CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken)
{
if (HostContext.RunMode == RunMode.Local)
{
return Task.FromResult<Timeline>(null);
}
CheckConnection();
return _taskClient.CreateTimelineAsync(scopeIdentifier, hubName, planId, new Timeline(timelineId), cancellationToken: cancellationToken);
}
public Task<List<TimelineRecord>> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable<TimelineRecord> records, CancellationToken cancellationToken)
{
if (HostContext.RunMode == RunMode.Local)
{
return Task.FromResult<List<TimelineRecord>>(null);
}
CheckConnection();
return _taskClient.UpdateTimelineRecordsAsync(scopeIdentifier, hubName, planId, timelineId, records, cancellationToken: cancellationToken);
}
public Task RaisePlanEventAsync<T>(Guid scopeIdentifier, string hubName, Guid planId, T eventData, CancellationToken cancellationToken) where T : JobEvent
{
if (HostContext.RunMode == RunMode.Local)
{
return Task.CompletedTask;
}
CheckConnection();
return _taskClient.RaisePlanEventAsync(scopeIdentifier, hubName, planId, eventData, cancellationToken: cancellationToken);
}
public Task<Timeline> GetTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken)
{
if (HostContext.RunMode == RunMode.Local)
{
return Task.FromResult<Timeline>(null);
}
CheckConnection();
return _taskClient.GetTimelineAsync(scopeIdentifier, hubName, planId, timelineId, includeRecords: true, cancellationToken: cancellationToken);
}
}
}

View File

@@ -0,0 +1,702 @@
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common.Util;
using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Pipelines = GitHub.DistributedTask.Pipelines;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common
{
[ServiceLocator(Default = typeof(JobServerQueue))]
public interface IJobServerQueue : IRunnerService, IThrottlingReporter
{
event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
Task ShutdownAsync();
void Start(Pipelines.AgentJobRequestMessage jobRequest);
void QueueWebConsoleLine(Guid stepRecordId, string line);
void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource);
void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord);
}
public sealed class JobServerQueue : RunnerService, IJobServerQueue
{
// Default delay for Dequeue process
private static readonly TimeSpan _aggressiveDelayForWebConsoleLineDequeue = TimeSpan.FromMilliseconds(250);
private static readonly TimeSpan _delayForWebConsoleLineDequeue = TimeSpan.FromMilliseconds(500);
private static readonly TimeSpan _delayForTimelineUpdateDequeue = TimeSpan.FromMilliseconds(500);
private static readonly TimeSpan _delayForFileUploadDequeue = TimeSpan.FromMilliseconds(1000);
// Job message information
private Guid _scopeIdentifier;
private string _hubName;
private Guid _planId;
private Guid _jobTimelineId;
private Guid _jobTimelineRecordId;
// queue for web console line
private readonly ConcurrentQueue<ConsoleLineInfo> _webConsoleLineQueue = new ConcurrentQueue<ConsoleLineInfo>();
// queue for file upload (log file or attachment)
private readonly ConcurrentQueue<UploadFileInfo> _fileUploadQueue = new ConcurrentQueue<UploadFileInfo>();
// queue for timeline or timeline record update (one queue per timeline)
private readonly ConcurrentDictionary<Guid, ConcurrentQueue<TimelineRecord>> _timelineUpdateQueue = new ConcurrentDictionary<Guid, ConcurrentQueue<TimelineRecord>>();
// indicate how many timelines we have, we will process _timelineUpdateQueue base on the order of timeline in this list
private readonly List<Guid> _allTimelines = new List<Guid>();
// bufferd timeline records that fail to update
private readonly Dictionary<Guid, List<TimelineRecord>> _bufferedRetryRecords = new Dictionary<Guid, List<TimelineRecord>>();
// Task for each queue's dequeue process
private Task _webConsoleLineDequeueTask;
private Task _fileUploadDequeueTask;
private Task _timelineUpdateDequeueTask;
// common
private IJobServer _jobServer;
private Task[] _allDequeueTasks;
private readonly TaskCompletionSource<int> _jobCompletionSource = new TaskCompletionSource<int>();
private bool _queueInProcess = false;
private ITerminal _term;
public event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
// Web console dequeue will start with process queue every 250ms for the first 60*4 times (~60 seconds).
// Then the dequeue will happen every 500ms.
// In this way, customer still can get instance live console output on job start,
// at the same time we can cut the load to server after the build run for more than 60s
private int _webConsoleLineAggressiveDequeueCount = 0;
private const int _webConsoleLineAggressiveDequeueLimit = 4 * 60;
private bool _webConsoleLineAggressiveDequeue = true;
private bool _firstConsoleOutputs = true;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_jobServer = hostContext.GetService<IJobServer>();
}
public void Start(Pipelines.AgentJobRequestMessage jobRequest)
{
Trace.Entering();
if (HostContext.RunMode == RunMode.Local)
{
_term = HostContext.GetService<ITerminal>();
return;
}
if (_queueInProcess)
{
Trace.Info("No-opt, all queue process tasks are running.");
return;
}
ArgUtil.NotNull(jobRequest, nameof(jobRequest));
ArgUtil.NotNull(jobRequest.Plan, nameof(jobRequest.Plan));
ArgUtil.NotNull(jobRequest.Timeline, nameof(jobRequest.Timeline));
_scopeIdentifier = jobRequest.Plan.ScopeIdentifier;
_hubName = jobRequest.Plan.PlanType;
_planId = jobRequest.Plan.PlanId;
_jobTimelineId = jobRequest.Timeline.Id;
_jobTimelineRecordId = jobRequest.JobId;
// Server already create the job timeline
_timelineUpdateQueue[_jobTimelineId] = new ConcurrentQueue<TimelineRecord>();
_allTimelines.Add(_jobTimelineId);
// Start three dequeue task
Trace.Info("Start process web console line queue.");
_webConsoleLineDequeueTask = ProcessWebConsoleLinesQueueAsync();
Trace.Info("Start process file upload queue.");
_fileUploadDequeueTask = ProcessFilesUploadQueueAsync();
Trace.Info("Start process timeline update queue.");
_timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync();
_allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask };
_queueInProcess = true;
}
// WebConsoleLine queue and FileUpload queue are always best effort
// TimelineUpdate queue error will become critical when timeline records contain output variabls.
public async Task ShutdownAsync()
{
if (HostContext.RunMode == RunMode.Local)
{
return;
}
if (!_queueInProcess)
{
Trace.Info("No-op, all queue process tasks have been stopped.");
}
Trace.Info("Fire signal to shutdown all queues.");
_jobCompletionSource.TrySetResult(0);
await Task.WhenAll(_allDequeueTasks);
_queueInProcess = false;
Trace.Info("All queue process task stopped.");
// Drain the queue
// ProcessWebConsoleLinesQueueAsync() will never throw exception, live console update is always best effort.
Trace.Verbose("Draining web console line queue.");
await ProcessWebConsoleLinesQueueAsync(runOnce: true);
Trace.Info("Web console line queue drained.");
// ProcessFilesUploadQueueAsync() will never throw exception, log file upload is always best effort.
Trace.Verbose("Draining file upload queue.");
await ProcessFilesUploadQueueAsync(runOnce: true);
Trace.Info("File upload queue drained.");
// ProcessTimelinesUpdateQueueAsync() will throw exception during shutdown
// if there is any timeline records that failed to update contains output variabls.
Trace.Verbose("Draining timeline update queue.");
await ProcessTimelinesUpdateQueueAsync(runOnce: true);
Trace.Info("Timeline update queue drained.");
Trace.Info("All queue process tasks have been stopped, and all queues are drained.");
}
public void QueueWebConsoleLine(Guid stepRecordId, string line)
{
Trace.Verbose("Enqueue web console line queue: {0}", line);
if (HostContext.RunMode == RunMode.Local)
{
if ((line ?? string.Empty).StartsWith("##[section]"))
{
Console.WriteLine("******************************************************************************");
Console.WriteLine(line.Substring("##[section]".Length));
Console.WriteLine("******************************************************************************");
}
else
{
Console.WriteLine(line);
}
return;
}
_webConsoleLineQueue.Enqueue(new ConsoleLineInfo(stepRecordId, line));
}
public void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource)
{
if (HostContext.RunMode == RunMode.Local)
{
return;
}
ArgUtil.NotEmpty(timelineId, nameof(timelineId));
ArgUtil.NotEmpty(timelineRecordId, nameof(timelineRecordId));
// all parameter not null, file path exist.
var newFile = new UploadFileInfo()
{
TimelineId = timelineId,
TimelineRecordId = timelineRecordId,
Type = type,
Name = name,
Path = path,
DeleteSource = deleteSource
};
Trace.Verbose("Enqueue file upload queue: file '{0}' attach to record {1}", newFile.Path, timelineRecordId);
_fileUploadQueue.Enqueue(newFile);
}
public void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord)
{
if (HostContext.RunMode == RunMode.Local)
{
return;
}
ArgUtil.NotEmpty(timelineId, nameof(timelineId));
ArgUtil.NotNull(timelineRecord, nameof(timelineRecord));
ArgUtil.NotEmpty(timelineRecord.Id, nameof(timelineRecord.Id));
_timelineUpdateQueue.TryAdd(timelineId, new ConcurrentQueue<TimelineRecord>());
Trace.Verbose("Enqueue timeline {0} update queue: {1}", timelineId, timelineRecord.Id);
_timelineUpdateQueue[timelineId].Enqueue(timelineRecord.Clone());
}
public void ReportThrottling(TimeSpan delay, DateTime expiration)
{
Trace.Info($"Receive server throttling report, expect delay {delay} milliseconds till {expiration}");
var throttlingEvent = JobServerQueueThrottling;
if (throttlingEvent != null)
{
throttlingEvent(this, new ThrottlingEventArgs(delay, expiration));
}
}
private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false)
{
while (!_jobCompletionSource.Task.IsCompleted || runOnce)
{
if (_webConsoleLineAggressiveDequeue && ++_webConsoleLineAggressiveDequeueCount > _webConsoleLineAggressiveDequeueLimit)
{
Trace.Info("Stop aggressive process web console line queue.");
_webConsoleLineAggressiveDequeue = false;
}
// Group consolelines by timeline record of each step
Dictionary<Guid, List<string>> stepsConsoleLines = new Dictionary<Guid, List<string>>();
List<Guid> stepRecordIds = new List<Guid>(); // We need to keep lines in order
int linesCounter = 0;
ConsoleLineInfo lineInfo;
while (_webConsoleLineQueue.TryDequeue(out lineInfo))
{
if (!stepsConsoleLines.ContainsKey(lineInfo.StepRecordId))
{
stepsConsoleLines[lineInfo.StepRecordId] = new List<string>();
stepRecordIds.Add(lineInfo.StepRecordId);
}
if (!string.IsNullOrEmpty(lineInfo.Line) && lineInfo.Line.Length > 1024)
{
Trace.Verbose("Web console line is more than 1024 chars, truncate to first 1024 chars");
lineInfo.Line = $"{lineInfo.Line.Substring(0, 1024)}...";
}
stepsConsoleLines[lineInfo.StepRecordId].Add(lineInfo.Line);
linesCounter++;
// process at most about 500 lines of web console line during regular timer dequeue task.
if (!runOnce && linesCounter > 500)
{
break;
}
}
// Batch post consolelines for each step timeline record
foreach (var stepRecordId in stepRecordIds)
{
// Split consolelines into batch, each batch will container at most 100 lines.
int batchCounter = 0;
List<List<string>> batchedLines = new List<List<string>>();
foreach (var line in stepsConsoleLines[stepRecordId])
{
var currentBatch = batchedLines.ElementAtOrDefault(batchCounter);
if (currentBatch == null)
{
batchedLines.Add(new List<string>());
currentBatch = batchedLines.ElementAt(batchCounter);
}
currentBatch.Add(line);
if (currentBatch.Count >= 100)
{
batchCounter++;
}
}
if (batchedLines.Count > 0)
{
// When job finish, web console lines becomes less interesting to customer
// We batch and produce 500 lines of web console output every 500ms
// If customer's task produce massive of outputs, then the last queue drain run might take forever.
// So we will only upload the last 200 lines of each step from all buffered web console lines.
if (runOnce && batchedLines.Count > 2)
{
Trace.Info($"Skip {batchedLines.Count - 2} batches web console lines for last run");
batchedLines = batchedLines.TakeLast(2).ToList();
batchedLines[0].Insert(0, "...");
}
int errorCount = 0;
foreach (var batch in batchedLines)
{
try
{
// we will not requeue failed batch, since the web console lines are time sensitive.
await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch, default(CancellationToken));
if (_firstConsoleOutputs)
{
HostContext.WritePerfCounter($"WorkerJobServerQueueAppendFirstConsoleOutput_{_planId.ToString()}");
_firstConsoleOutputs = false;
}
}
catch (Exception ex)
{
Trace.Info("Catch exception during append web console line, keep going since the process is best effort.");
Trace.Error(ex);
errorCount++;
}
}
Trace.Info("Try to append {0} batches web console lines for record '{2}', success rate: {1}/{0}.", batchedLines.Count, batchedLines.Count - errorCount, stepRecordId);
}
}
if (runOnce)
{
break;
}
else
{
await Task.Delay(_webConsoleLineAggressiveDequeue ? _aggressiveDelayForWebConsoleLineDequeue : _delayForWebConsoleLineDequeue);
}
}
}
private async Task ProcessFilesUploadQueueAsync(bool runOnce = false)
{
while (!_jobCompletionSource.Task.IsCompleted || runOnce)
{
List<UploadFileInfo> filesToUpload = new List<UploadFileInfo>();
UploadFileInfo dequeueFile;
while (_fileUploadQueue.TryDequeue(out dequeueFile))
{
filesToUpload.Add(dequeueFile);
// process at most 10 file upload.
if (!runOnce && filesToUpload.Count > 10)
{
break;
}
}
if (filesToUpload.Count > 0)
{
if (runOnce)
{
Trace.Info($"Uploading {filesToUpload.Count} files in one shot.");
}
// TODO: upload all file in parallel
int errorCount = 0;
foreach (var file in filesToUpload)
{
try
{
await UploadFile(file);
}
catch (Exception ex)
{
Trace.Info("Catch exception during log or attachment file upload, keep going since the process is best effort.");
Trace.Error(ex);
errorCount++;
// put the failed upload file back to queue.
// TODO: figure out how should we retry paging log upload.
//lock (_fileUploadQueueLock)
//{
// _fileUploadQueue.Enqueue(file);
//}
}
}
Trace.Info("Try to upload {0} log files or attachments, success rate: {1}/{0}.", filesToUpload.Count, filesToUpload.Count - errorCount);
}
if (runOnce)
{
break;
}
else
{
await Task.Delay(_delayForFileUploadDequeue);
}
}
}
private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false)
{
while (!_jobCompletionSource.Task.IsCompleted || runOnce)
{
List<PendingTimelineRecord> pendingUpdates = new List<PendingTimelineRecord>();
foreach (var timeline in _allTimelines)
{
ConcurrentQueue<TimelineRecord> recordQueue;
if (_timelineUpdateQueue.TryGetValue(timeline, out recordQueue))
{
List<TimelineRecord> records = new List<TimelineRecord>();
TimelineRecord record;
while (recordQueue.TryDequeue(out record))
{
records.Add(record);
// process at most 25 timeline records update for each timeline.
if (!runOnce && records.Count > 25)
{
break;
}
}
if (records.Count > 0)
{
pendingUpdates.Add(new PendingTimelineRecord() { TimelineId = timeline, PendingRecords = records.ToList() });
}
}
}
// we need track whether we have new sub-timeline been created on the last run.
// if so, we need continue update timeline record even we on the last run.
bool pendingSubtimelineUpdate = false;
List<Exception> mainTimelineRecordsUpdateErrors = new List<Exception>();
if (pendingUpdates.Count > 0)
{
foreach (var update in pendingUpdates)
{
List<TimelineRecord> bufferedRecords;
if (_bufferedRetryRecords.TryGetValue(update.TimelineId, out bufferedRecords))
{
update.PendingRecords.InsertRange(0, bufferedRecords);
}
update.PendingRecords = MergeTimelineRecords(update.PendingRecords);
foreach (var detailTimeline in update.PendingRecords.Where(r => r.Details != null))
{
if (!_allTimelines.Contains(detailTimeline.Details.Id))
{
try
{
Timeline newTimeline = await _jobServer.CreateTimelineAsync(_scopeIdentifier, _hubName, _planId, detailTimeline.Details.Id, default(CancellationToken));
_allTimelines.Add(newTimeline.Id);
pendingSubtimelineUpdate = true;
}
catch (TimelineExistsException)
{
Trace.Info("Catch TimelineExistsException during timeline creation. Ignore the error since server already had this timeline.");
_allTimelines.Add(detailTimeline.Details.Id);
}
catch (Exception ex)
{
Trace.Error(ex);
}
}
}
try
{
await _jobServer.UpdateTimelineRecordsAsync(_scopeIdentifier, _hubName, _planId, update.TimelineId, update.PendingRecords, default(CancellationToken));
if (_bufferedRetryRecords.Remove(update.TimelineId))
{
Trace.Verbose("Cleanup buffered timeline record for timeline: {0}.", update.TimelineId);
}
}
catch (Exception ex)
{
Trace.Info("Catch exception during update timeline records, try to update these timeline records next time.");
Trace.Error(ex);
_bufferedRetryRecords[update.TimelineId] = update.PendingRecords.ToList();
if (update.TimelineId == _jobTimelineId)
{
mainTimelineRecordsUpdateErrors.Add(ex);
}
}
}
}
if (runOnce)
{
// continue process timeline records update,
// we might have more records need update,
// since we just create a new sub-timeline
if (pendingSubtimelineUpdate)
{
continue;
}
else
{
if (mainTimelineRecordsUpdateErrors.Count > 0 &&
_bufferedRetryRecords.ContainsKey(_jobTimelineId) &&
_bufferedRetryRecords[_jobTimelineId] != null &&
_bufferedRetryRecords[_jobTimelineId].Any(r => r.Variables.Count > 0))
{
Trace.Info("Fail to update timeline records with output variables. Throw exception to fail the job since output variables are critical to downstream jobs.");
throw new AggregateException("Failed to publish output variables.", mainTimelineRecordsUpdateErrors);
}
else
{
break;
}
}
}
else
{
await Task.Delay(_delayForTimelineUpdateDequeue);
}
}
}
private List<TimelineRecord> MergeTimelineRecords(List<TimelineRecord> timelineRecords)
{
if (timelineRecords == null || timelineRecords.Count <= 1)
{
return timelineRecords;
}
Dictionary<Guid, TimelineRecord> dict = new Dictionary<Guid, TimelineRecord>();
foreach (TimelineRecord rec in timelineRecords)
{
if (rec == null)
{
continue;
}
TimelineRecord timelineRecord;
if (dict.TryGetValue(rec.Id, out timelineRecord))
{
// Merge rec into timelineRecord
timelineRecord.CurrentOperation = rec.CurrentOperation ?? timelineRecord.CurrentOperation;
timelineRecord.Details = rec.Details ?? timelineRecord.Details;
timelineRecord.FinishTime = rec.FinishTime ?? timelineRecord.FinishTime;
timelineRecord.Log = rec.Log ?? timelineRecord.Log;
timelineRecord.Name = rec.Name ?? timelineRecord.Name;
timelineRecord.RefName = rec.RefName ?? timelineRecord.RefName;
timelineRecord.PercentComplete = rec.PercentComplete ?? timelineRecord.PercentComplete;
timelineRecord.RecordType = rec.RecordType ?? timelineRecord.RecordType;
timelineRecord.Result = rec.Result ?? timelineRecord.Result;
timelineRecord.ResultCode = rec.ResultCode ?? timelineRecord.ResultCode;
timelineRecord.StartTime = rec.StartTime ?? timelineRecord.StartTime;
timelineRecord.State = rec.State ?? timelineRecord.State;
timelineRecord.WorkerName = rec.WorkerName ?? timelineRecord.WorkerName;
if (rec.ErrorCount != null && rec.ErrorCount > 0)
{
timelineRecord.ErrorCount = rec.ErrorCount;
}
if (rec.WarningCount != null && rec.WarningCount > 0)
{
timelineRecord.WarningCount = rec.WarningCount;
}
if (rec.Issues.Count > 0)
{
timelineRecord.Issues.Clear();
timelineRecord.Issues.AddRange(rec.Issues.Select(i => i.Clone()));
}
if (rec.Variables.Count > 0)
{
foreach (var variable in rec.Variables)
{
timelineRecord.Variables[variable.Key] = variable.Value.Clone();
}
}
}
else
{
dict.Add(rec.Id, rec);
}
}
var mergedRecords = dict.Values.ToList();
Trace.Verbose("Merged Timeline records");
foreach (var record in mergedRecords)
{
Trace.Verbose($" Record: t={record.RecordType}, n={record.Name}, s={record.State}, st={record.StartTime}, {record.PercentComplete}%, ft={record.FinishTime}, r={record.Result}: {record.CurrentOperation}");
if (record.Issues != null && record.Issues.Count > 0)
{
foreach (var issue in record.Issues)
{
String source;
issue.Data.TryGetValue("sourcepath", out source);
Trace.Verbose($" Issue: c={issue.Category}, t={issue.Type}, s={source ?? string.Empty}, m={issue.Message}");
}
}
if (record.Variables != null && record.Variables.Count > 0)
{
foreach (var variable in record.Variables)
{
Trace.Verbose($" Variable: n={variable.Key}, secret={variable.Value.IsSecret}");
}
}
}
return mergedRecords;
}
private async Task UploadFile(UploadFileInfo file)
{
bool uploadSucceed = false;
try
{
if (String.Equals(file.Type, CoreAttachmentType.Log, StringComparison.OrdinalIgnoreCase))
{
// Create the log
var taskLog = await _jobServer.CreateLogAsync(_scopeIdentifier, _hubName, _planId, new TaskLog(String.Format(@"logs\{0:D}", file.TimelineRecordId)), default(CancellationToken));
// Upload the contents
using (FileStream fs = File.Open(file.Path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
var logUploaded = await _jobServer.AppendLogContentAsync(_scopeIdentifier, _hubName, _planId, taskLog.Id, fs, default(CancellationToken));
}
// Create a new record and only set the Log field
var attachmentUpdataRecord = new TimelineRecord() { Id = file.TimelineRecordId, Log = taskLog };
QueueTimelineRecordUpdate(file.TimelineId, attachmentUpdataRecord);
}
else
{
// Create attachment
using (FileStream fs = File.Open(file.Path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
var result = await _jobServer.CreateAttachmentAsync(_scopeIdentifier, _hubName, _planId, file.TimelineId, file.TimelineRecordId, file.Type, file.Name, fs, default(CancellationToken));
}
}
uploadSucceed = true;
}
finally
{
if (uploadSucceed && file.DeleteSource)
{
try
{
File.Delete(file.Path);
}
catch (Exception ex)
{
Trace.Info("Catch exception during delete success uploaded file.");
Trace.Error(ex);
}
}
}
}
}
internal class PendingTimelineRecord
{
public Guid TimelineId { get; set; }
public List<TimelineRecord> PendingRecords { get; set; }
}
internal class UploadFileInfo
{
public Guid TimelineId { get; set; }
public Guid TimelineRecordId { get; set; }
public string Type { get; set; }
public string Name { get; set; }
public string Path { get; set; }
public bool DeleteSource { get; set; }
}
internal class ConsoleLineInfo
{
public ConsoleLineInfo(Guid recordId, string line)
{
this.StepRecordId = recordId;
this.Line = line;
}
public Guid StepRecordId { get; set; }
public string Line { get; set; }
}
}

View File

@@ -0,0 +1,61 @@
using System;
using System.Threading.Tasks;
using GitHub.Services.WebApi;
using GitHub.Services.Location.Client;
using GitHub.Services.Location;
namespace GitHub.Runner.Common
{
[ServiceLocator(Default = typeof(LocationServer))]
public interface ILocationServer : IRunnerService
{
Task ConnectAsync(VssConnection jobConnection);
Task<ConnectionData> GetConnectionDataAsync();
}
public sealed class LocationServer : RunnerService, ILocationServer
{
private bool _hasConnection;
private VssConnection _connection;
private LocationHttpClient _locationClient;
public async Task ConnectAsync(VssConnection jobConnection)
{
_connection = jobConnection;
int attemptCount = 5;
while (!_connection.HasAuthenticated && attemptCount-- > 0)
{
try
{
await _connection.ConnectAsync();
break;
}
catch (Exception ex) when (attemptCount > 0)
{
Trace.Info($"Catch exception during connect. {attemptCount} attempt left.");
Trace.Error(ex);
}
await Task.Delay(100);
}
_locationClient = _connection.GetClient<LocationHttpClient>();
_hasConnection = true;
}
private void CheckConnection()
{
if (!_hasConnection)
{
throw new InvalidOperationException("SetConnection");
}
}
public async Task<ConnectionData> GetConnectionDataAsync()
{
CheckConnection();
return await _locationClient.GetConnectionDataAsync(ConnectOptions.None, 0);
}
}
}

View File

@@ -0,0 +1,124 @@
using GitHub.Runner.Common.Util;
using System;
using System.IO;
namespace GitHub.Runner.Common
{
[ServiceLocator(Default = typeof(PagingLogger))]
public interface IPagingLogger : IRunnerService
{
long TotalLines { get; }
void Setup(Guid timelineId, Guid timelineRecordId);
void Write(string message);
void End();
}
public class PagingLogger : RunnerService, IPagingLogger
{
public static string PagingFolder = "pages";
// 8 MB
public const int PageSize = 8 * 1024 * 1024;
private Guid _timelineId;
private Guid _timelineRecordId;
private string _pageId;
private FileStream _pageData;
private StreamWriter _pageWriter;
private int _byteCount;
private int _pageCount;
private long _totalLines;
private string _dataFileName;
private string _pagesFolder;
private IJobServerQueue _jobServerQueue;
public long TotalLines => _totalLines;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_totalLines = 0;
_pageId = Guid.NewGuid().ToString();
_pagesFolder = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Diag), PagingFolder);
_jobServerQueue = HostContext.GetService<IJobServerQueue>();
Directory.CreateDirectory(_pagesFolder);
}
public void Setup(Guid timelineId, Guid timelineRecordId)
{
_timelineId = timelineId;
_timelineRecordId = timelineRecordId;
}
//
// Write a metadata file with id etc, point to pages on disk.
// Each page is a guid_#. As a page rolls over, it events it's done
// and the consumer queues it for upload
// Ensure this is lazy. Create a page on first write
//
public void Write(string message)
{
// lazy creation on write
if (_pageWriter == null)
{
Create();
}
string line = $"{DateTime.UtcNow.ToString("O")} {message}";
_pageWriter.WriteLine(line);
_totalLines++;
if (line.IndexOf('\n') != -1)
{
foreach (char c in line)
{
if (c == '\n')
{
_totalLines++;
}
}
}
_byteCount += System.Text.Encoding.UTF8.GetByteCount(line);
if (_byteCount >= PageSize)
{
NewPage();
}
}
public void End()
{
EndPage();
}
private void Create()
{
NewPage();
}
private void NewPage()
{
EndPage();
_byteCount = 0;
_dataFileName = Path.Combine(_pagesFolder, $"{_pageId}_{++_pageCount}.log");
_pageData = new FileStream(_dataFileName, FileMode.CreateNew);
_pageWriter = new StreamWriter(_pageData, System.Text.Encoding.UTF8);
}
private void EndPage()
{
if (_pageWriter != null)
{
_pageWriter.Flush();
_pageData.Flush();
//The StreamWriter object calls Dispose() on the provided Stream object when StreamWriter.Dispose is called.
_pageWriter.Dispose();
_pageWriter = null;
_pageData = null;
_jobServerQueue.QueueFileUpload(_timelineId, _timelineRecordId, "DistributedTask.Core.Log", "CustomToolLog", _dataFileName, true);
}
}
}
}

View File

@@ -0,0 +1,100 @@
using System;
using System.IO;
using System.IO.Pipes;
using System.Threading;
using System.Threading.Tasks;
namespace GitHub.Runner.Common
{
public delegate void StartProcessDelegate(string pipeHandleOut, string pipeHandleIn);
public enum MessageType
{
NotInitialized = -1,
NewJobRequest = 1,
CancelRequest = 2,
RunnerShutdown = 3,
OperatingSystemShutdown = 4
}
public struct WorkerMessage
{
public MessageType MessageType;
public string Body;
public WorkerMessage(MessageType messageType, string body)
{
MessageType = messageType;
Body = body;
}
}
[ServiceLocator(Default = typeof(ProcessChannel))]
public interface IProcessChannel : IDisposable, IRunnerService
{
void StartServer(StartProcessDelegate startProcess);
void StartClient(string pipeNameInput, string pipeNameOutput);
Task SendAsync(MessageType messageType, string body, CancellationToken cancellationToken);
Task<WorkerMessage> ReceiveAsync(CancellationToken cancellationToken);
}
public sealed class ProcessChannel : RunnerService, IProcessChannel
{
private AnonymousPipeServerStream _inServer;
private AnonymousPipeServerStream _outServer;
private AnonymousPipeClientStream _inClient;
private AnonymousPipeClientStream _outClient;
private StreamString _writeStream;
private StreamString _readStream;
public void StartServer(StartProcessDelegate startProcess)
{
_outServer = new AnonymousPipeServerStream(PipeDirection.Out, HandleInheritability.Inheritable);
_inServer = new AnonymousPipeServerStream(PipeDirection.In, HandleInheritability.Inheritable);
_readStream = new StreamString(_inServer);
_writeStream = new StreamString(_outServer);
startProcess(_outServer.GetClientHandleAsString(), _inServer.GetClientHandleAsString());
_outServer.DisposeLocalCopyOfClientHandle();
_inServer.DisposeLocalCopyOfClientHandle();
}
public void StartClient(string pipeNameInput, string pipeNameOutput)
{
_inClient = new AnonymousPipeClientStream(PipeDirection.In, pipeNameInput);
_outClient = new AnonymousPipeClientStream(PipeDirection.Out, pipeNameOutput);
_readStream = new StreamString(_inClient);
_writeStream = new StreamString(_outClient);
}
public async Task SendAsync(MessageType messageType, string body, CancellationToken cancellationToken)
{
await _writeStream.WriteInt32Async((int)messageType, cancellationToken);
await _writeStream.WriteStringAsync(body, cancellationToken);
}
public async Task<WorkerMessage> ReceiveAsync(CancellationToken cancellationToken)
{
WorkerMessage result = new WorkerMessage(MessageType.NotInitialized, string.Empty);
result.MessageType = (MessageType)await _readStream.ReadInt32Async(cancellationToken);
result.Body = await _readStream.ReadStringAsync(cancellationToken);
return result;
}
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
private void Dispose(bool disposing)
{
if (disposing)
{
_inServer?.Dispose();
_outServer?.Dispose();
_inClient?.Dispose();
_outClient?.Dispose();
}
}
}
}

View File

@@ -0,0 +1,396 @@
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace GitHub.Runner.Common
{
#if OS_WINDOWS
public static class WindowsProcessExtensions
{
// Reference: https://blogs.msdn.microsoft.com/matt_pietrek/2004/08/25/reading-another-processs-environment/
// Reference: http://blog.gapotchenko.com/eazfuscator.net/reading-environment-variables
public static string GetEnvironmentVariable(this Process process, IHostContext hostContext, string variable)
{
var trace = hostContext.GetTrace(nameof(WindowsProcessExtensions));
Dictionary<string, string> environmentVariables = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
IntPtr processHandle = process.SafeHandle.DangerousGetHandle();
IntPtr environmentBlockAddress;
if (Environment.Is64BitOperatingSystem)
{
PROCESS_BASIC_INFORMATION64 pbi = new PROCESS_BASIC_INFORMATION64();
int returnLength = 0;
int status = NtQueryInformationProcess64(processHandle, PROCESSINFOCLASS.ProcessBasicInformation, ref pbi, Marshal.SizeOf(pbi), ref returnLength);
if (status != 0)
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
bool wow64;
if (!IsWow64Process(processHandle, out wow64))
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
if (!wow64)
{
// 64 bits process on 64 bits OS
IntPtr UserProcessParameterAddress = ReadIntPtr64(processHandle, new IntPtr(pbi.PebBaseAddress) + 0x20);
environmentBlockAddress = ReadIntPtr64(processHandle, UserProcessParameterAddress + 0x80);
}
else
{
// 32 bits process on 64 bits OS
IntPtr UserProcessParameterAddress = ReadIntPtr32(processHandle, new IntPtr(pbi.PebBaseAddress) + 0x1010);
environmentBlockAddress = ReadIntPtr32(processHandle, UserProcessParameterAddress + 0x48);
}
}
else
{
PROCESS_BASIC_INFORMATION32 pbi = new PROCESS_BASIC_INFORMATION32();
int returnLength = 0;
int status = NtQueryInformationProcess32(processHandle, PROCESSINFOCLASS.ProcessBasicInformation, ref pbi, Marshal.SizeOf(pbi), ref returnLength);
if (status != 0)
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
// 32 bits process on 32 bits OS
IntPtr UserProcessParameterAddress = ReadIntPtr32(processHandle, new IntPtr(pbi.PebBaseAddress) + 0x10);
environmentBlockAddress = ReadIntPtr32(processHandle, UserProcessParameterAddress + 0x48);
}
MEMORY_BASIC_INFORMATION memInfo = new MEMORY_BASIC_INFORMATION();
if (VirtualQueryEx(processHandle, environmentBlockAddress, ref memInfo, Marshal.SizeOf(memInfo)) == 0)
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
Int64 dataSize = memInfo.RegionSize.ToInt64() - (environmentBlockAddress.ToInt64() - memInfo.BaseAddress.ToInt64());
byte[] envData = new byte[dataSize];
IntPtr res_len = IntPtr.Zero;
if (!ReadProcessMemory(processHandle, environmentBlockAddress, envData, new IntPtr(dataSize), ref res_len))
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
if (res_len.ToInt64() != dataSize)
{
throw new ArgumentOutOfRangeException(nameof(ReadProcessMemory));
}
string environmentVariableString;
Int64 environmentVariableBytesLength = 0;
// check env encoding
if (envData[0] != 0 && envData[1] == 0)
{
// Unicode
for (Int64 index = 0; index < dataSize; index++)
{
// Unicode encoded environment variables block ends up with '\0\0\0\0'.
if (environmentVariableBytesLength == 0 &&
envData[index] == 0 &&
index + 3 < dataSize &&
envData[index + 1] == 0 &&
envData[index + 2] == 0 &&
envData[index + 3] == 0)
{
environmentVariableBytesLength = index + 3;
}
else if (environmentVariableBytesLength != 0)
{
// set it '\0' so we can easily trim it, most array method doesn't take int64
envData[index] = 0;
}
}
if (environmentVariableBytesLength == 0)
{
throw new ArgumentException(nameof(environmentVariableBytesLength));
}
environmentVariableString = Encoding.Unicode.GetString(envData);
}
else if (envData[0] != 0 && envData[1] != 0)
{
// ANSI
for (Int64 index = 0; index < dataSize; index++)
{
// Unicode encoded environment variables block ends up with '\0\0'.
if (environmentVariableBytesLength == 0 &&
envData[index] == 0 &&
index + 1 < dataSize &&
envData[index + 1] == 0)
{
environmentVariableBytesLength = index + 1;
}
else if (environmentVariableBytesLength != 0)
{
// set it '\0' so we can easily trim it, most array method doesn't take int64
envData[index] = 0;
}
}
if (environmentVariableBytesLength == 0)
{
throw new ArgumentException(nameof(environmentVariableBytesLength));
}
environmentVariableString = Encoding.Default.GetString(envData);
}
else
{
throw new ArgumentException(nameof(envData));
}
foreach (var envString in environmentVariableString.Split("\0", StringSplitOptions.RemoveEmptyEntries))
{
string[] env = envString.Split("=", 2);
if (!string.IsNullOrEmpty(env[0]))
{
environmentVariables[env[0]] = env[1];
trace.Verbose($"PID:{process.Id} ({env[0]}={env[1]})");
}
}
if (environmentVariables.TryGetValue(variable, out string envVariable))
{
return envVariable;
}
else
{
return null;
}
}
private static IntPtr ReadIntPtr32(IntPtr hProcess, IntPtr ptr)
{
IntPtr readPtr = IntPtr.Zero;
IntPtr data = Marshal.AllocHGlobal(sizeof(Int32));
try
{
IntPtr res_len = IntPtr.Zero;
if (!ReadProcessMemory(hProcess, ptr, data, new IntPtr(sizeof(Int32)), ref res_len))
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
if (res_len.ToInt32() != sizeof(Int32))
{
throw new ArgumentOutOfRangeException(nameof(ReadProcessMemory));
}
readPtr = new IntPtr(Marshal.ReadInt32(data));
}
finally
{
Marshal.FreeHGlobal(data);
}
return readPtr;
}
private static IntPtr ReadIntPtr64(IntPtr hProcess, IntPtr ptr)
{
IntPtr readPtr = IntPtr.Zero;
IntPtr data = Marshal.AllocHGlobal(IntPtr.Size);
try
{
IntPtr res_len = IntPtr.Zero;
if (!ReadProcessMemory(hProcess, ptr, data, new IntPtr(sizeof(Int64)), ref res_len))
{
throw new Win32Exception(Marshal.GetLastWin32Error());
}
if (res_len.ToInt32() != IntPtr.Size)
{
throw new ArgumentOutOfRangeException(nameof(ReadProcessMemory));
}
readPtr = Marshal.ReadIntPtr(data);
}
finally
{
Marshal.FreeHGlobal(data);
}
return readPtr;
}
private enum PROCESSINFOCLASS : int
{
ProcessBasicInformation = 0
};
[StructLayout(LayoutKind.Sequential)]
private struct MEMORY_BASIC_INFORMATION
{
public IntPtr BaseAddress;
public IntPtr AllocationBase;
public int AllocationProtect;
public IntPtr RegionSize;
public int State;
public int Protect;
public int Type;
}
[StructLayout(LayoutKind.Sequential)]
private struct PROCESS_BASIC_INFORMATION64
{
public long ExitStatus;
public long PebBaseAddress;
public long AffinityMask;
public long BasePriority;
public long UniqueProcessId;
public long InheritedFromUniqueProcessId;
};
[StructLayout(LayoutKind.Sequential)]
private struct PROCESS_BASIC_INFORMATION32
{
public int ExitStatus;
public int PebBaseAddress;
public int AffinityMask;
public int BasePriority;
public int UniqueProcessId;
public int InheritedFromUniqueProcessId;
};
[DllImport("ntdll.dll", SetLastError = true, EntryPoint = "NtQueryInformationProcess")]
private static extern int NtQueryInformationProcess64(IntPtr processHandle, PROCESSINFOCLASS processInformationClass, ref PROCESS_BASIC_INFORMATION64 processInformation, int processInformationLength, ref int returnLength);
[DllImport("ntdll.dll", SetLastError = true, EntryPoint = "NtQueryInformationProcess")]
private static extern int NtQueryInformationProcess32(IntPtr processHandle, PROCESSINFOCLASS processInformationClass, ref PROCESS_BASIC_INFORMATION32 processInformation, int processInformationLength, ref int returnLength);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool IsWow64Process(IntPtr processHandle, out bool wow64Process);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool ReadProcessMemory(IntPtr hProcess, IntPtr lpBaseAddress, IntPtr lpBuffer, IntPtr dwSize, ref IntPtr lpNumberOfBytesRead);
[DllImport("kernel32.dll", SetLastError = true)]
private static extern bool ReadProcessMemory(IntPtr hProcess, IntPtr lpBaseAddress, [Out] byte[] lpBuffer, IntPtr dwSize, ref IntPtr lpNumberOfBytesRead);
[DllImport("kernel32.dll")]
private static extern int VirtualQueryEx(IntPtr processHandle, IntPtr baseAddress, ref MEMORY_BASIC_INFORMATION memoryInformation, int memoryInformationLength);
}
#else
public static class LinuxProcessExtensions
{
public static string GetEnvironmentVariable(this Process process, IHostContext hostContext, string variable)
{
var trace = hostContext.GetTrace(nameof(LinuxProcessExtensions));
Dictionary<string, string> env = new Dictionary<string, string>();
if (Directory.Exists("/proc"))
{
string envFile = $"/proc/{process.Id}/environ";
trace.Info($"Read env from {envFile}");
string envContent = File.ReadAllText(envFile);
if (!string.IsNullOrEmpty(envContent))
{
// on linux, environment variables are seprated by '\0'
var envList = envContent.Split('\0', StringSplitOptions.RemoveEmptyEntries);
foreach (var envStr in envList)
{
// split on the first '='
var keyValuePair = envStr.Split('=', 2);
if (keyValuePair.Length == 2)
{
env[keyValuePair[0]] = keyValuePair[1];
trace.Verbose($"PID:{process.Id} ({keyValuePair[0]}={keyValuePair[1]})");
}
}
}
}
else
{
// On OSX, there is no /proc folder for us to read environment for given process,
// So we have call `ps e -p <pid> -o command` to print out env to STDOUT,
// However, the output env are not format in a parseable way, it's just a string that concatenate all envs with space,
// It doesn't escape '=' or ' ', so we can't parse the output into a dictionary of all envs.
// So we only look for the env you request, in the format of variable=value. (it won't work if you variable contains = or space)
trace.Info($"Read env from output of `ps e -p {process.Id} -o command`");
List<string> psOut = new List<string>();
object outputLock = new object();
using (var p = hostContext.CreateService<IProcessInvoker>())
{
p.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (outputLock)
{
psOut.Add(stdout.Data);
}
}
};
p.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (outputLock)
{
trace.Error(stderr.Data);
}
}
};
int exitCode = p.ExecuteAsync(workingDirectory: hostContext.GetDirectory(WellKnownDirectory.Root),
fileName: "ps",
arguments: $"e -p {process.Id} -o command",
environment: null,
cancellationToken: CancellationToken.None).GetAwaiter().GetResult();
if (exitCode == 0)
{
trace.Info($"Successfully dump environment variables for {process.Id}");
if (psOut.Count > 0)
{
string psOutputString = string.Join(" ", psOut);
trace.Verbose($"ps output: '{psOutputString}'");
int varStartIndex = psOutputString.IndexOf(variable, StringComparison.Ordinal);
if (varStartIndex >= 0)
{
string rightPart = psOutputString.Substring(varStartIndex + variable.Length + 1);
if (rightPart.IndexOf(' ') > 0)
{
string value = rightPart.Substring(0, rightPart.IndexOf(' '));
env[variable] = value;
}
else
{
env[variable] = rightPart;
}
trace.Verbose($"PID:{process.Id} ({variable}={env[variable]})");
}
}
}
}
}
if (env.TryGetValue(variable, out string envVariable))
{
return envVariable;
}
else
{
return null;
}
}
}
#endif
}

View File

@@ -0,0 +1,329 @@
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks;
namespace GitHub.Runner.Common
{
[ServiceLocator(Default = typeof(ProcessInvokerWrapper))]
public interface IProcessInvoker : IDisposable, IRunnerService
{
event EventHandler<ProcessDataReceivedEventArgs> OutputDataReceived;
event EventHandler<ProcessDataReceivedEventArgs> ErrorDataReceived;
Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
CancellationToken cancellationToken);
Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
CancellationToken cancellationToken);
Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
CancellationToken cancellationToken);
Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
CancellationToken cancellationToken);
Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
Channel<string> redirectStandardIn,
CancellationToken cancellationToken);
Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
Channel<string> redirectStandardIn,
bool inheritConsoleHandler,
CancellationToken cancellationToken);
Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
Channel<string> redirectStandardIn,
bool inheritConsoleHandler,
bool keepStandardInOpen,
CancellationToken cancellationToken);
Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
Channel<string> redirectStandardIn,
bool inheritConsoleHandler,
bool keepStandardInOpen,
bool highPriorityProcess,
CancellationToken cancellationToken);
}
// The implementation of the process invoker does not hook up DataReceivedEvent and ErrorReceivedEvent of Process,
// instead, we read both STDOUT and STDERR stream manually on seperate thread.
// The reason is we find a huge perf issue about process STDOUT/STDERR with those events.
//
// Missing functionalities:
// 1. Cancel/Kill process tree
// 2. Make sure STDOUT and STDERR not process out of order
public sealed class ProcessInvokerWrapper : RunnerService, IProcessInvoker
{
private ProcessInvoker _invoker;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_invoker = new ProcessInvoker(Trace);
}
public event EventHandler<ProcessDataReceivedEventArgs> OutputDataReceived;
public event EventHandler<ProcessDataReceivedEventArgs> ErrorDataReceived;
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: false,
cancellationToken: cancellationToken);
}
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: requireExitCodeZero,
outputEncoding: null,
cancellationToken: cancellationToken);
}
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: requireExitCodeZero,
outputEncoding: outputEncoding,
killProcessOnCancel: false,
cancellationToken: cancellationToken);
}
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: requireExitCodeZero,
outputEncoding: outputEncoding,
killProcessOnCancel: killProcessOnCancel,
redirectStandardIn: null,
cancellationToken: cancellationToken);
}
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
Channel<string> redirectStandardIn,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: requireExitCodeZero,
outputEncoding: outputEncoding,
killProcessOnCancel: killProcessOnCancel,
redirectStandardIn: redirectStandardIn,
inheritConsoleHandler: false,
cancellationToken: cancellationToken
);
}
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
Channel<string> redirectStandardIn,
bool inheritConsoleHandler,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: requireExitCodeZero,
outputEncoding: outputEncoding,
killProcessOnCancel: killProcessOnCancel,
redirectStandardIn: redirectStandardIn,
inheritConsoleHandler: inheritConsoleHandler,
keepStandardInOpen: false,
cancellationToken: cancellationToken
);
}
public Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
Channel<string> redirectStandardIn,
bool inheritConsoleHandler,
bool keepStandardInOpen,
CancellationToken cancellationToken)
{
return ExecuteAsync(
workingDirectory: workingDirectory,
fileName: fileName,
arguments: arguments,
environment: environment,
requireExitCodeZero: requireExitCodeZero,
outputEncoding: outputEncoding,
killProcessOnCancel: killProcessOnCancel,
redirectStandardIn: redirectStandardIn,
inheritConsoleHandler: inheritConsoleHandler,
keepStandardInOpen: keepStandardInOpen,
highPriorityProcess: false,
cancellationToken: cancellationToken
);
}
public async Task<int> ExecuteAsync(
string workingDirectory,
string fileName,
string arguments,
IDictionary<string, string> environment,
bool requireExitCodeZero,
Encoding outputEncoding,
bool killProcessOnCancel,
Channel<string> redirectStandardIn,
bool inheritConsoleHandler,
bool keepStandardInOpen,
bool highPriorityProcess,
CancellationToken cancellationToken)
{
_invoker.ErrorDataReceived += this.ErrorDataReceived;
_invoker.OutputDataReceived += this.OutputDataReceived;
return await _invoker.ExecuteAsync(
workingDirectory,
fileName,
arguments,
environment,
requireExitCodeZero,
outputEncoding,
killProcessOnCancel,
redirectStandardIn,
inheritConsoleHandler,
keepStandardInOpen,
highPriorityProcess,
cancellationToken);
}
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
private void Dispose(bool disposing)
{
if (disposing)
{
if (_invoker != null)
{
_invoker.Dispose();
_invoker = null;
}
}
}
}
}

View File

@@ -0,0 +1,68 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp2.2</TargetFramework>
<OutputType>Library</OutputType>
<RuntimeIdentifiers>win-x64;win-x86;linux-x64;linux-arm;rhel.6-x64;osx-x64</RuntimeIdentifiers>
<TargetLatestRuntimePatch>true</TargetLatestRuntimePatch>
<AssetTargetFallback>portable-net45+win8</AssetTargetFallback>
<NoWarn>NU1701;NU1603</NoWarn>
<Version>$(Version)</Version>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Sdk\Sdk.csproj" />
<ProjectReference Include="..\Runner.Sdk\Runner.Sdk.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Win32.Registry" Version="4.4.0" />
<PackageReference Include="Newtonsoft.Json" Version="11.0.2" />
<PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
<PackageReference Include="System.Text.Encoding.CodePages" Version="4.4.0" />
<PackageReference Include="System.Threading.Channels" Version="4.4.0" />
</ItemGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
<DebugType>portable</DebugType>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true' AND '$(PackageRuntime)' == 'win-x64'">
<DefineConstants>OS_WINDOWS;X64;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true' AND '$(PackageRuntime)' == 'win-x86'">
<DefineConstants>OS_WINDOWS;X86;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true' AND '$(Configuration)' == 'Debug' AND '$(PackageRuntime)' == 'win-x64'">
<DefineConstants>OS_WINDOWS;X64;DEBUG;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true' AND '$(Configuration)' == 'Debug' AND '$(PackageRuntime)' == 'win-x86'">
<DefineConstants>OS_WINDOWS;X86;DEBUG;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'">
<DefineConstants>OS_OSX;X64;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true' AND '$(Configuration)' == 'Debug'">
<DefineConstants>OS_OSX;DEBUG;X64;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true' AND '$(PackageRuntime)' == 'linux-x64'">
<DefineConstants>OS_LINUX;X64;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true' AND '$(PackageRuntime)' == 'rhel.6-x64'">
<DefineConstants>OS_LINUX;OS_RHEL6;X64;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true' AND '$(PackageRuntime)' == 'linux-arm'">
<DefineConstants>OS_LINUX;ARM;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true' AND '$(Configuration)' == 'Debug' AND '$(PackageRuntime)' == 'linux-x64'">
<DefineConstants>OS_LINUX;X64;DEBUG;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true' AND '$(Configuration)' == 'Debug' AND '$(PackageRuntime)' == 'rhel.6-x64'">
<DefineConstants>OS_LINUX;OS_RHEL6;X64;DEBUG;TRACE</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true' AND '$(Configuration)' == 'Debug' AND '$(PackageRuntime)' == 'linux-arm'">
<DefineConstants>OS_LINUX;ARM;DEBUG;TRACE</DefineConstants>
</PropertyGroup>
</Project>

View File

@@ -0,0 +1,231 @@
using System;
using GitHub.Runner.Common.Util;
using System.IO;
using System.Runtime.Serialization;
using GitHub.Services.Common;
using System.Security.Cryptography.X509Certificates;
using System.Net;
using System.Net.Security;
using System.Net.Http;
using GitHub.Services.WebApi;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common
{
[ServiceLocator(Default = typeof(RunnerCertificateManager))]
public interface IRunnerCertificateManager : IRunnerService
{
bool SkipServerCertificateValidation { get; }
string CACertificateFile { get; }
string ClientCertificateFile { get; }
string ClientCertificatePrivateKeyFile { get; }
string ClientCertificateArchiveFile { get; }
string ClientCertificatePassword { get; }
IVssClientCertificateManager VssClientCertificateManager { get; }
}
public class RunnerCertificateManager : RunnerService, IRunnerCertificateManager
{
private RunnerClientCertificateManager _runnerClientCertificateManager = new RunnerClientCertificateManager();
public bool SkipServerCertificateValidation { private set; get; }
public string CACertificateFile { private set; get; }
public string ClientCertificateFile { private set; get; }
public string ClientCertificatePrivateKeyFile { private set; get; }
public string ClientCertificateArchiveFile { private set; get; }
public string ClientCertificatePassword { private set; get; }
public IVssClientCertificateManager VssClientCertificateManager => _runnerClientCertificateManager;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
LoadCertificateSettings();
}
// This should only be called from config
public void SetupCertificate(bool skipCertValidation, string caCert, string clientCert, string clientCertPrivateKey, string clientCertArchive, string clientCertPassword)
{
Trace.Info("Setup runner certificate setting base on configuration inputs.");
if (skipCertValidation)
{
Trace.Info("Ignore SSL server certificate validation error");
SkipServerCertificateValidation = true;
VssClientHttpRequestSettings.Default.ServerCertificateValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
}
if (!string.IsNullOrEmpty(caCert))
{
ArgUtil.File(caCert, nameof(caCert));
Trace.Info($"Self-Signed CA '{caCert}'");
}
if (!string.IsNullOrEmpty(clientCert))
{
ArgUtil.File(clientCert, nameof(clientCert));
ArgUtil.File(clientCertPrivateKey, nameof(clientCertPrivateKey));
ArgUtil.File(clientCertArchive, nameof(clientCertArchive));
Trace.Info($"Client cert '{clientCert}'");
Trace.Info($"Client cert private key '{clientCertPrivateKey}'");
Trace.Info($"Client cert archive '{clientCertArchive}'");
}
CACertificateFile = caCert;
ClientCertificateFile = clientCert;
ClientCertificatePrivateKeyFile = clientCertPrivateKey;
ClientCertificateArchiveFile = clientCertArchive;
ClientCertificatePassword = clientCertPassword;
_runnerClientCertificateManager.AddClientCertificate(ClientCertificateArchiveFile, ClientCertificatePassword);
}
// This should only be called from config
public void SaveCertificateSetting()
{
string certSettingFile = HostContext.GetConfigFile(WellKnownConfigFile.Certificates);
IOUtil.DeleteFile(certSettingFile);
var setting = new RunnerCertificateSetting();
if (SkipServerCertificateValidation)
{
Trace.Info($"Store Skip ServerCertificateValidation setting to '{certSettingFile}'");
setting.SkipServerCertValidation = true;
}
if (!string.IsNullOrEmpty(CACertificateFile))
{
Trace.Info($"Store CA cert setting to '{certSettingFile}'");
setting.CACert = CACertificateFile;
}
if (!string.IsNullOrEmpty(ClientCertificateFile) &&
!string.IsNullOrEmpty(ClientCertificatePrivateKeyFile) &&
!string.IsNullOrEmpty(ClientCertificateArchiveFile))
{
Trace.Info($"Store client cert settings to '{certSettingFile}'");
setting.ClientCert = ClientCertificateFile;
setting.ClientCertPrivatekey = ClientCertificatePrivateKeyFile;
setting.ClientCertArchive = ClientCertificateArchiveFile;
if (!string.IsNullOrEmpty(ClientCertificatePassword))
{
string lookupKey = Guid.NewGuid().ToString("D").ToUpperInvariant();
Trace.Info($"Store client cert private key password with lookup key {lookupKey}");
var credStore = HostContext.GetService<IRunnerCredentialStore>();
credStore.Write($"GITHUB_ACTIONS_RUNNER_CLIENT_CERT_PASSWORD_{lookupKey}", "GitHub", ClientCertificatePassword);
setting.ClientCertPasswordLookupKey = lookupKey;
}
}
if (SkipServerCertificateValidation ||
!string.IsNullOrEmpty(CACertificateFile) ||
!string.IsNullOrEmpty(ClientCertificateFile))
{
IOUtil.SaveObject(setting, certSettingFile);
File.SetAttributes(certSettingFile, File.GetAttributes(certSettingFile) | FileAttributes.Hidden);
}
}
// This should only be called from unconfig
public void DeleteCertificateSetting()
{
string certSettingFile = HostContext.GetConfigFile(WellKnownConfigFile.Certificates);
if (File.Exists(certSettingFile))
{
Trace.Info($"Load runner certificate setting from '{certSettingFile}'");
var certSetting = IOUtil.LoadObject<RunnerCertificateSetting>(certSettingFile);
if (certSetting != null && !string.IsNullOrEmpty(certSetting.ClientCertPasswordLookupKey))
{
Trace.Info("Delete client cert private key password from credential store.");
var credStore = HostContext.GetService<IRunnerCredentialStore>();
credStore.Delete($"GITHUB_ACTIONS_RUNNER_CLIENT_CERT_PASSWORD_{certSetting.ClientCertPasswordLookupKey}");
}
Trace.Info($"Delete cert setting file: {certSettingFile}");
IOUtil.DeleteFile(certSettingFile);
}
}
public void LoadCertificateSettings()
{
string certSettingFile = HostContext.GetConfigFile(WellKnownConfigFile.Certificates);
if (File.Exists(certSettingFile))
{
Trace.Info($"Load runner certificate setting from '{certSettingFile}'");
var certSetting = IOUtil.LoadObject<RunnerCertificateSetting>(certSettingFile);
ArgUtil.NotNull(certSetting, nameof(RunnerCertificateSetting));
if (certSetting.SkipServerCertValidation)
{
Trace.Info("Ignore SSL server certificate validation error");
SkipServerCertificateValidation = true;
VssClientHttpRequestSettings.Default.ServerCertificateValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator;
}
if (!string.IsNullOrEmpty(certSetting.CACert))
{
// make sure all settings file exist
ArgUtil.File(certSetting.CACert, nameof(certSetting.CACert));
Trace.Info($"CA '{certSetting.CACert}'");
CACertificateFile = certSetting.CACert;
}
if (!string.IsNullOrEmpty(certSetting.ClientCert))
{
// make sure all settings file exist
ArgUtil.File(certSetting.ClientCert, nameof(certSetting.ClientCert));
ArgUtil.File(certSetting.ClientCertPrivatekey, nameof(certSetting.ClientCertPrivatekey));
ArgUtil.File(certSetting.ClientCertArchive, nameof(certSetting.ClientCertArchive));
Trace.Info($"Client cert '{certSetting.ClientCert}'");
Trace.Info($"Client cert private key '{certSetting.ClientCertPrivatekey}'");
Trace.Info($"Client cert archive '{certSetting.ClientCertArchive}'");
ClientCertificateFile = certSetting.ClientCert;
ClientCertificatePrivateKeyFile = certSetting.ClientCertPrivatekey;
ClientCertificateArchiveFile = certSetting.ClientCertArchive;
if (!string.IsNullOrEmpty(certSetting.ClientCertPasswordLookupKey))
{
var cerdStore = HostContext.GetService<IRunnerCredentialStore>();
ClientCertificatePassword = cerdStore.Read($"GITHUB_ACTIONS_RUNNER_CLIENT_CERT_PASSWORD_{certSetting.ClientCertPasswordLookupKey}").Password;
HostContext.SecretMasker.AddValue(ClientCertificatePassword);
}
_runnerClientCertificateManager.AddClientCertificate(ClientCertificateArchiveFile, ClientCertificatePassword);
}
}
else
{
Trace.Info("No certificate setting found.");
}
}
}
[DataContract]
internal class RunnerCertificateSetting
{
[DataMember]
public bool SkipServerCertValidation { get; set; }
[DataMember]
public string CACert { get; set; }
[DataMember]
public string ClientCert { get; set; }
[DataMember]
public string ClientCertPrivatekey { get; set; }
[DataMember]
public string ClientCertArchive { get; set; }
[DataMember]
public string ClientCertPasswordLookupKey { get; set; }
}
}

View File

@@ -0,0 +1,948 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Net;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using GitHub.Runner.Common.Util;
using Newtonsoft.Json;
using System.IO;
using System.Runtime.Serialization;
using System.Security.Cryptography;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common
{
// The purpose of this class is to store user's credential during runner configuration and retrive the credential back at runtime.
#if OS_WINDOWS
[ServiceLocator(Default = typeof(WindowsRunnerCredentialStore))]
#elif OS_OSX
[ServiceLocator(Default = typeof(MacOSRunnerCredentialStore))]
#else
[ServiceLocator(Default = typeof(LinuxRunnerCredentialStore))]
#endif
public interface IRunnerCredentialStore : IRunnerService
{
NetworkCredential Write(string target, string username, string password);
// throw exception when target not found from cred store
NetworkCredential Read(string target);
// throw exception when target not found from cred store
void Delete(string target);
}
#if OS_WINDOWS
// Windows credential store is per user.
// This is a limitation for user configure the runner run as windows service, when user's current login account is different with the service run as account.
// Ex: I login the box as domain\admin, configure the runner as windows service and run as domian\buildserver
// domain\buildserver won't read the stored credential from domain\admin's windows credential store.
// To workaround this limitation.
// Anytime we try to save a credential:
// 1. store it into current user's windows credential store
// 2. use DP-API do a machine level encrypt and store the encrypted content on disk.
// At the first time we try to read the credential:
// 1. read from current user's windows credential store, delete the DP-API encrypted backup content on disk if the windows credential store read succeed.
// 2. if credential not found in current user's windows credential store, read from the DP-API encrypted backup content on disk,
// write the credential back the current user's windows credential store and delete the backup on disk.
public sealed class WindowsRunnerCredentialStore : RunnerService, IRunnerCredentialStore
{
private string _credStoreFile;
private Dictionary<string, string> _credStore;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_credStoreFile = hostContext.GetConfigFile(WellKnownConfigFile.CredentialStore);
if (File.Exists(_credStoreFile))
{
_credStore = IOUtil.LoadObject<Dictionary<string, string>>(_credStoreFile);
}
else
{
_credStore = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
}
}
public NetworkCredential Write(string target, string username, string password)
{
Trace.Entering();
ArgUtil.NotNullOrEmpty(target, nameof(target));
ArgUtil.NotNullOrEmpty(username, nameof(username));
ArgUtil.NotNullOrEmpty(password, nameof(password));
// save to .credential_store file first, then Windows credential store
string usernameBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(username));
string passwordBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(password));
// Base64Username:Base64Password -> DP-API machine level encrypt -> Base64Encoding
string encryptedUsernamePassword = Convert.ToBase64String(ProtectedData.Protect(Encoding.UTF8.GetBytes($"{usernameBase64}:{passwordBase64}"), null, DataProtectionScope.LocalMachine));
Trace.Info($"Credentials for '{target}' written to credential store file.");
_credStore[target] = encryptedUsernamePassword;
// save to .credential_store file
SyncCredentialStoreFile();
// save to Windows Credential Store
return WriteInternal(target, username, password);
}
public NetworkCredential Read(string target)
{
Trace.Entering();
ArgUtil.NotNullOrEmpty(target, nameof(target));
IntPtr credPtr = IntPtr.Zero;
try
{
if (CredRead(target, CredentialType.Generic, 0, out credPtr))
{
Credential credStruct = (Credential)Marshal.PtrToStructure(credPtr, typeof(Credential));
int passwordLength = (int)credStruct.CredentialBlobSize;
string password = passwordLength > 0 ? Marshal.PtrToStringUni(credStruct.CredentialBlob, passwordLength / sizeof(char)) : String.Empty;
string username = Marshal.PtrToStringUni(credStruct.UserName);
Trace.Info($"Credentials for '{target}' read from windows credential store.");
// delete from .credential_store file since we are able to read it from windows credential store
if (_credStore.Remove(target))
{
Trace.Info($"Delete credentials for '{target}' from credential store file.");
SyncCredentialStoreFile();
}
return new NetworkCredential(username, password);
}
else
{
// Can't read from Windows Credential Store, fail back to .credential_store file
if (_credStore.ContainsKey(target) && !string.IsNullOrEmpty(_credStore[target]))
{
Trace.Info($"Credentials for '{target}' read from credential store file.");
// Base64Decode -> DP-API machine level decrypt -> Base64Username:Base64Password -> Base64Decode
string decryptedUsernamePassword = Encoding.UTF8.GetString(ProtectedData.Unprotect(Convert.FromBase64String(_credStore[target]), null, DataProtectionScope.LocalMachine));
string[] credential = decryptedUsernamePassword.Split(':');
if (credential.Length == 2 && !string.IsNullOrEmpty(credential[0]) && !string.IsNullOrEmpty(credential[1]))
{
string username = Encoding.UTF8.GetString(Convert.FromBase64String(credential[0]));
string password = Encoding.UTF8.GetString(Convert.FromBase64String(credential[1]));
// store back to windows credential store for current user
NetworkCredential creds = WriteInternal(target, username, password);
// delete from .credential_store file since we are able to write the credential to windows credential store for current user.
if (_credStore.Remove(target))
{
Trace.Info($"Delete credentials for '{target}' from credential store file.");
SyncCredentialStoreFile();
}
return creds;
}
else
{
throw new ArgumentOutOfRangeException(nameof(decryptedUsernamePassword));
}
}
throw new Win32Exception(Marshal.GetLastWin32Error(), $"CredRead throw an error for '{target}'");
}
}
finally
{
if (credPtr != IntPtr.Zero)
{
CredFree(credPtr);
}
}
}
public void Delete(string target)
{
Trace.Entering();
ArgUtil.NotNullOrEmpty(target, nameof(target));
// remove from .credential_store file
if (_credStore.Remove(target))
{
Trace.Info($"Delete credentials for '{target}' from credential store file.");
SyncCredentialStoreFile();
}
// remove from windows credential store
if (!CredDelete(target, CredentialType.Generic, 0))
{
throw new Win32Exception(Marshal.GetLastWin32Error(), $"Failed to delete credentials for {target}");
}
else
{
Trace.Info($"Credentials for '{target}' deleted from windows credential store.");
}
}
private NetworkCredential WriteInternal(string target, string username, string password)
{
// save to Windows Credential Store
Credential credential = new Credential()
{
Type = CredentialType.Generic,
Persist = (UInt32)CredentialPersist.LocalMachine,
TargetName = Marshal.StringToCoTaskMemUni(target),
UserName = Marshal.StringToCoTaskMemUni(username),
CredentialBlob = Marshal.StringToCoTaskMemUni(password),
CredentialBlobSize = (UInt32)Encoding.Unicode.GetByteCount(password),
AttributeCount = 0,
Comment = IntPtr.Zero,
Attributes = IntPtr.Zero,
TargetAlias = IntPtr.Zero
};
try
{
if (CredWrite(ref credential, 0))
{
Trace.Info($"Credentials for '{target}' written to windows credential store.");
return new NetworkCredential(username, password);
}
else
{
int error = Marshal.GetLastWin32Error();
throw new Win32Exception(error, "Failed to write credentials");
}
}
finally
{
if (credential.CredentialBlob != IntPtr.Zero)
{
Marshal.FreeCoTaskMem(credential.CredentialBlob);
}
if (credential.TargetName != IntPtr.Zero)
{
Marshal.FreeCoTaskMem(credential.TargetName);
}
if (credential.UserName != IntPtr.Zero)
{
Marshal.FreeCoTaskMem(credential.UserName);
}
}
}
private void SyncCredentialStoreFile()
{
Trace.Info("Sync in-memory credential store with credential store file.");
// delete the cred store file first anyway, since it's a readonly file.
IOUtil.DeleteFile(_credStoreFile);
// delete cred store file when all creds gone
if (_credStore.Count == 0)
{
return;
}
else
{
IOUtil.SaveObject(_credStore, _credStoreFile);
File.SetAttributes(_credStoreFile, File.GetAttributes(_credStoreFile) | FileAttributes.Hidden);
}
}
[DllImport("Advapi32.dll", EntryPoint = "CredDeleteW", CharSet = CharSet.Unicode, SetLastError = true)]
internal static extern bool CredDelete(string target, CredentialType type, int reservedFlag);
[DllImport("Advapi32.dll", EntryPoint = "CredReadW", CharSet = CharSet.Unicode, SetLastError = true)]
internal static extern bool CredRead(string target, CredentialType type, int reservedFlag, out IntPtr CredentialPtr);
[DllImport("Advapi32.dll", EntryPoint = "CredWriteW", CharSet = CharSet.Unicode, SetLastError = true)]
internal static extern bool CredWrite([In] ref Credential userCredential, [In] UInt32 flags);
[DllImport("Advapi32.dll", EntryPoint = "CredFree", SetLastError = true)]
internal static extern bool CredFree([In] IntPtr cred);
internal enum CredentialPersist : UInt32
{
Session = 0x01,
LocalMachine = 0x02
}
internal enum CredentialType : uint
{
Generic = 0x01,
DomainPassword = 0x02,
DomainCertificate = 0x03
}
[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
internal struct Credential
{
public UInt32 Flags;
public CredentialType Type;
public IntPtr TargetName;
public IntPtr Comment;
public System.Runtime.InteropServices.ComTypes.FILETIME LastWritten;
public UInt32 CredentialBlobSize;
public IntPtr CredentialBlob;
public UInt32 Persist;
public UInt32 AttributeCount;
public IntPtr Attributes;
public IntPtr TargetAlias;
public IntPtr UserName;
}
}
#elif OS_OSX
public sealed class MacOSRunnerCredentialStore : RunnerService, IRunnerCredentialStore
{
private const string _osxRunnerCredStoreKeyChainName = "_GITHUB_ACTIONS_RUNNER_CREDSTORE_INTERNAL_";
// Keychain requires a password, but this is not intended to add security
private const string _osxRunnerCredStoreKeyChainPassword = "C46F23C36AF94B72B1EAEE32C68670A0";
private string _securityUtil;
private string _runnerCredStoreKeyChain;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_securityUtil = WhichUtil.Which("security", true, Trace);
_runnerCredStoreKeyChain = hostContext.GetConfigFile(WellKnownConfigFile.CredentialStore);
// Create osx key chain if it doesn't exists.
if (!File.Exists(_runnerCredStoreKeyChain))
{
List<string> securityOut = new List<string>();
List<string> securityError = new List<string>();
object outputLock = new object();
using (var p = HostContext.CreateService<IProcessInvoker>())
{
p.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (outputLock)
{
securityOut.Add(stdout.Data);
}
}
};
p.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (outputLock)
{
securityError.Add(stderr.Data);
}
}
};
// make sure the 'security' has access to the key so we won't get prompt at runtime.
int exitCode = p.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root),
fileName: _securityUtil,
arguments: $"create-keychain -p {_osxRunnerCredStoreKeyChainPassword} \"{_runnerCredStoreKeyChain}\"",
environment: null,
cancellationToken: CancellationToken.None).GetAwaiter().GetResult();
if (exitCode == 0)
{
Trace.Info($"Successfully create-keychain for {_runnerCredStoreKeyChain}");
}
else
{
if (securityOut.Count > 0)
{
Trace.Error(string.Join(Environment.NewLine, securityOut));
}
if (securityError.Count > 0)
{
Trace.Error(string.Join(Environment.NewLine, securityError));
}
throw new InvalidOperationException($"'security create-keychain' failed with exit code {exitCode}.");
}
}
}
else
{
// Try unlock and lock the keychain, make sure it's still in good stage
UnlockKeyChain();
LockKeyChain();
}
}
public NetworkCredential Write(string target, string username, string password)
{
Trace.Entering();
ArgUtil.NotNullOrEmpty(target, nameof(target));
ArgUtil.NotNullOrEmpty(username, nameof(username));
ArgUtil.NotNullOrEmpty(password, nameof(password));
try
{
UnlockKeyChain();
// base64encode username + ':' + base64encode password
// OSX keychain requires you provide -s target and -a username to retrieve password
// So, we will trade both username and password as 'secret' store into keychain
string usernameBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(username));
string passwordBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(password));
string secretForKeyChain = $"{usernameBase64}:{passwordBase64}";
List<string> securityOut = new List<string>();
List<string> securityError = new List<string>();
object outputLock = new object();
using (var p = HostContext.CreateService<IProcessInvoker>())
{
p.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (outputLock)
{
securityOut.Add(stdout.Data);
}
}
};
p.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (outputLock)
{
securityError.Add(stderr.Data);
}
}
};
// make sure the 'security' has access to the key so we won't get prompt at runtime.
int exitCode = p.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root),
fileName: _securityUtil,
arguments: $"add-generic-password -s {target} -a GITHUBACTIONSRUNNER -w {secretForKeyChain} -T \"{_securityUtil}\" \"{_runnerCredStoreKeyChain}\"",
environment: null,
cancellationToken: CancellationToken.None).GetAwaiter().GetResult();
if (exitCode == 0)
{
Trace.Info($"Successfully add-generic-password for {target} (GITHUBACTIONSRUNNER)");
}
else
{
if (securityOut.Count > 0)
{
Trace.Error(string.Join(Environment.NewLine, securityOut));
}
if (securityError.Count > 0)
{
Trace.Error(string.Join(Environment.NewLine, securityError));
}
throw new InvalidOperationException($"'security add-generic-password' failed with exit code {exitCode}.");
}
}
return new NetworkCredential(username, password);
}
finally
{
LockKeyChain();
}
}
public NetworkCredential Read(string target)
{
Trace.Entering();
ArgUtil.NotNullOrEmpty(target, nameof(target));
try
{
UnlockKeyChain();
string username;
string password;
List<string> securityOut = new List<string>();
List<string> securityError = new List<string>();
object outputLock = new object();
using (var p = HostContext.CreateService<IProcessInvoker>())
{
p.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (outputLock)
{
securityOut.Add(stdout.Data);
}
}
};
p.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (outputLock)
{
securityError.Add(stderr.Data);
}
}
};
int exitCode = p.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root),
fileName: _securityUtil,
arguments: $"find-generic-password -s {target} -a GITHUBACTIONSRUNNER -w -g \"{_runnerCredStoreKeyChain}\"",
environment: null,
cancellationToken: CancellationToken.None).GetAwaiter().GetResult();
if (exitCode == 0)
{
string keyChainSecret = securityOut.First();
string[] secrets = keyChainSecret.Split(':');
if (secrets.Length == 2 && !string.IsNullOrEmpty(secrets[0]) && !string.IsNullOrEmpty(secrets[1]))
{
Trace.Info($"Successfully find-generic-password for {target} (GITHUBACTIONSRUNNER)");
username = Encoding.UTF8.GetString(Convert.FromBase64String(secrets[0]));
password = Encoding.UTF8.GetString(Convert.FromBase64String(secrets[1]));
return new NetworkCredential(username, password);
}
else
{
throw new ArgumentOutOfRangeException(nameof(keyChainSecret));
}
}
else
{
if (securityOut.Count > 0)
{
Trace.Error(string.Join(Environment.NewLine, securityOut));
}
if (securityError.Count > 0)
{
Trace.Error(string.Join(Environment.NewLine, securityError));
}
throw new InvalidOperationException($"'security find-generic-password' failed with exit code {exitCode}.");
}
}
}
finally
{
LockKeyChain();
}
}
public void Delete(string target)
{
Trace.Entering();
ArgUtil.NotNullOrEmpty(target, nameof(target));
try
{
UnlockKeyChain();
List<string> securityOut = new List<string>();
List<string> securityError = new List<string>();
object outputLock = new object();
using (var p = HostContext.CreateService<IProcessInvoker>())
{
p.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (outputLock)
{
securityOut.Add(stdout.Data);
}
}
};
p.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (outputLock)
{
securityError.Add(stderr.Data);
}
}
};
int exitCode = p.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root),
fileName: _securityUtil,
arguments: $"delete-generic-password -s {target} -a GITHUBACTIONSRUNNER \"{_runnerCredStoreKeyChain}\"",
environment: null,
cancellationToken: CancellationToken.None).GetAwaiter().GetResult();
if (exitCode == 0)
{
Trace.Info($"Successfully delete-generic-password for {target} (GITHUBACTIONSRUNNER)");
}
else
{
if (securityOut.Count > 0)
{
Trace.Error(string.Join(Environment.NewLine, securityOut));
}
if (securityError.Count > 0)
{
Trace.Error(string.Join(Environment.NewLine, securityError));
}
throw new InvalidOperationException($"'security delete-generic-password' failed with exit code {exitCode}.");
}
}
}
finally
{
LockKeyChain();
}
}
private void UnlockKeyChain()
{
Trace.Entering();
ArgUtil.NotNullOrEmpty(_securityUtil, nameof(_securityUtil));
ArgUtil.NotNullOrEmpty(_runnerCredStoreKeyChain, nameof(_runnerCredStoreKeyChain));
List<string> securityOut = new List<string>();
List<string> securityError = new List<string>();
object outputLock = new object();
using (var p = HostContext.CreateService<IProcessInvoker>())
{
p.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (outputLock)
{
securityOut.Add(stdout.Data);
}
}
};
p.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (outputLock)
{
securityError.Add(stderr.Data);
}
}
};
// make sure the 'security' has access to the key so we won't get prompt at runtime.
int exitCode = p.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root),
fileName: _securityUtil,
arguments: $"unlock-keychain -p {_osxRunnerCredStoreKeyChainPassword} \"{_runnerCredStoreKeyChain}\"",
environment: null,
cancellationToken: CancellationToken.None).GetAwaiter().GetResult();
if (exitCode == 0)
{
Trace.Info($"Successfully unlock-keychain for {_runnerCredStoreKeyChain}");
}
else
{
if (securityOut.Count > 0)
{
Trace.Error(string.Join(Environment.NewLine, securityOut));
}
if (securityError.Count > 0)
{
Trace.Error(string.Join(Environment.NewLine, securityError));
}
throw new InvalidOperationException($"'security unlock-keychain' failed with exit code {exitCode}.");
}
}
}
private void LockKeyChain()
{
Trace.Entering();
ArgUtil.NotNullOrEmpty(_securityUtil, nameof(_securityUtil));
ArgUtil.NotNullOrEmpty(_runnerCredStoreKeyChain, nameof(_runnerCredStoreKeyChain));
List<string> securityOut = new List<string>();
List<string> securityError = new List<string>();
object outputLock = new object();
using (var p = HostContext.CreateService<IProcessInvoker>())
{
p.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout)
{
if (!string.IsNullOrEmpty(stdout.Data))
{
lock (outputLock)
{
securityOut.Add(stdout.Data);
}
}
};
p.ErrorDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stderr)
{
if (!string.IsNullOrEmpty(stderr.Data))
{
lock (outputLock)
{
securityError.Add(stderr.Data);
}
}
};
// make sure the 'security' has access to the key so we won't get prompt at runtime.
int exitCode = p.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Root),
fileName: _securityUtil,
arguments: $"lock-keychain \"{_runnerCredStoreKeyChain}\"",
environment: null,
cancellationToken: CancellationToken.None).GetAwaiter().GetResult();
if (exitCode == 0)
{
Trace.Info($"Successfully lock-keychain for {_runnerCredStoreKeyChain}");
}
else
{
if (securityOut.Count > 0)
{
Trace.Error(string.Join(Environment.NewLine, securityOut));
}
if (securityError.Count > 0)
{
Trace.Error(string.Join(Environment.NewLine, securityError));
}
throw new InvalidOperationException($"'security lock-keychain' failed with exit code {exitCode}.");
}
}
}
}
#else
public sealed class LinuxRunnerCredentialStore : RunnerService, IRunnerCredentialStore
{
// 'ghrunner' 128 bits iv
private readonly byte[] iv = new byte[] { 0x67, 0x68, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x67, 0x68, 0x72, 0x75, 0x6e, 0x6e, 0x65, 0x72 };
// 256 bits key
private byte[] _symmetricKey;
private string _credStoreFile;
private Dictionary<string, Credential> _credStore;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_credStoreFile = hostContext.GetConfigFile(WellKnownConfigFile.CredentialStore);
if (File.Exists(_credStoreFile))
{
_credStore = IOUtil.LoadObject<Dictionary<string, Credential>>(_credStoreFile);
}
else
{
_credStore = new Dictionary<string, Credential>(StringComparer.OrdinalIgnoreCase);
}
string machineId;
if (File.Exists("/etc/machine-id"))
{
// try use machine-id as encryption key
// this helps avoid accidental information disclosure, but isn't intended for true security
machineId = File.ReadAllLines("/etc/machine-id").FirstOrDefault();
Trace.Info($"machine-id length {machineId?.Length ?? 0}.");
// machine-id doesn't exist or machine-id is not 256 bits
if (string.IsNullOrEmpty(machineId) || machineId.Length != 32)
{
Trace.Warning("Can not get valid machine id from '/etc/machine-id'.");
machineId = "43e7fe5da07740cf914b90f1dac51c2a";
}
}
else
{
// /etc/machine-id not exist
Trace.Warning("/etc/machine-id doesn't exist.");
machineId = "43e7fe5da07740cf914b90f1dac51c2a";
}
List<byte> keyBuilder = new List<byte>();
foreach (var c in machineId)
{
keyBuilder.Add(Convert.ToByte(c));
}
_symmetricKey = keyBuilder.ToArray();
}
public NetworkCredential Write(string target, string username, string password)
{
Trace.Entering();
ArgUtil.NotNullOrEmpty(target, nameof(target));
ArgUtil.NotNullOrEmpty(username, nameof(username));
ArgUtil.NotNullOrEmpty(password, nameof(password));
Trace.Info($"Store credential for '{target}' to cred store.");
Credential cred = new Credential(username, Encrypt(password));
_credStore[target] = cred;
SyncCredentialStoreFile();
return new NetworkCredential(username, password);
}
public NetworkCredential Read(string target)
{
Trace.Entering();
ArgUtil.NotNullOrEmpty(target, nameof(target));
Trace.Info($"Read credential for '{target}' from cred store.");
if (_credStore.ContainsKey(target))
{
Credential cred = _credStore[target];
if (!string.IsNullOrEmpty(cred.UserName) && !string.IsNullOrEmpty(cred.Password))
{
Trace.Info($"Return credential for '{target}' from cred store.");
return new NetworkCredential(cred.UserName, Decrypt(cred.Password));
}
}
throw new KeyNotFoundException(target);
}
public void Delete(string target)
{
Trace.Entering();
ArgUtil.NotNullOrEmpty(target, nameof(target));
if (_credStore.ContainsKey(target))
{
Trace.Info($"Delete credential for '{target}' from cred store.");
_credStore.Remove(target);
SyncCredentialStoreFile();
}
else
{
throw new KeyNotFoundException(target);
}
}
private void SyncCredentialStoreFile()
{
Trace.Entering();
Trace.Info("Sync in-memory credential store with credential store file.");
// delete cred store file when all creds gone
if (_credStore.Count == 0)
{
IOUtil.DeleteFile(_credStoreFile);
return;
}
if (!File.Exists(_credStoreFile))
{
CreateCredentialStoreFile();
}
IOUtil.SaveObject(_credStore, _credStoreFile);
}
private string Encrypt(string secret)
{
using (Aes aes = Aes.Create())
{
aes.Key = _symmetricKey;
aes.IV = iv;
// Create a decrytor to perform the stream transform.
ICryptoTransform encryptor = aes.CreateEncryptor();
// Create the streams used for encryption.
using (MemoryStream msEncrypt = new MemoryStream())
{
using (CryptoStream csEncrypt = new CryptoStream(msEncrypt, encryptor, CryptoStreamMode.Write))
{
using (StreamWriter swEncrypt = new StreamWriter(csEncrypt))
{
swEncrypt.Write(secret);
}
return Convert.ToBase64String(msEncrypt.ToArray());
}
}
}
}
private string Decrypt(string encryptedText)
{
using (Aes aes = Aes.Create())
{
aes.Key = _symmetricKey;
aes.IV = iv;
// Create a decrytor to perform the stream transform.
ICryptoTransform decryptor = aes.CreateDecryptor();
// Create the streams used for decryption.
using (MemoryStream msDecrypt = new MemoryStream(Convert.FromBase64String(encryptedText)))
{
using (CryptoStream csDecrypt = new CryptoStream(msDecrypt, decryptor, CryptoStreamMode.Read))
{
using (StreamReader srDecrypt = new StreamReader(csDecrypt))
{
// Read the decrypted bytes from the decrypting stream and place them in a string.
return srDecrypt.ReadToEnd();
}
}
}
}
}
private void CreateCredentialStoreFile()
{
File.WriteAllText(_credStoreFile, "");
File.SetAttributes(_credStoreFile, File.GetAttributes(_credStoreFile) | FileAttributes.Hidden);
// Try to lock down the .credentials_store file to the owner/group
var chmodPath = WhichUtil.Which("chmod", trace: Trace);
if (!String.IsNullOrEmpty(chmodPath))
{
var arguments = $"600 {new FileInfo(_credStoreFile).FullName}";
using (var invoker = HostContext.CreateService<IProcessInvoker>())
{
var exitCode = invoker.ExecuteAsync(HostContext.GetDirectory(WellKnownDirectory.Root), chmodPath, arguments, null, default(CancellationToken)).GetAwaiter().GetResult();
if (exitCode == 0)
{
Trace.Info("Successfully set permissions for credentials store file {0}", _credStoreFile);
}
else
{
Trace.Warning("Unable to successfully set permissions for credentials store file {0}. Received exit code {1} from {2}", _credStoreFile, exitCode, chmodPath);
}
}
}
else
{
Trace.Warning("Unable to locate chmod to set permissions for credentials store file {0}.", _credStoreFile);
}
}
}
[DataContract]
internal class Credential
{
public Credential()
{ }
public Credential(string userName, string password)
{
UserName = userName;
Password = password;
}
[DataMember(IsRequired = true)]
public string UserName { get; set; }
[DataMember(IsRequired = true)]
public string Password { get; set; }
}
#endif
}

View File

@@ -0,0 +1,355 @@
using GitHub.DistributedTask.WebApi;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Runner.Common.Util;
using GitHub.Services.WebApi;
using GitHub.Services.Common;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common
{
public enum RunnerConnectionType
{
Generic,
MessageQueue,
JobRequest
}
[ServiceLocator(Default = typeof(RunnerServer))]
public interface IRunnerServer : IRunnerService
{
Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
Task RefreshConnectionAsync(RunnerConnectionType connectionType, TimeSpan timeout);
void SetConnectionTimeout(RunnerConnectionType connectionType, TimeSpan timeout);
// Configuration
Task<TaskAgent> AddAgentAsync(Int32 agentPoolId, TaskAgent agent);
Task DeleteAgentAsync(int agentPoolId, int agentId);
Task<List<TaskAgentPool>> GetAgentPoolsAsync(string agentPoolName = null, TaskAgentPoolType poolType = TaskAgentPoolType.Automation);
Task<List<TaskAgent>> GetAgentsAsync(int agentPoolId, string agentName = null);
Task<TaskAgent> UpdateAgentAsync(int agentPoolId, TaskAgent agent);
// messagequeue
Task<TaskAgentSession> CreateAgentSessionAsync(Int32 poolId, TaskAgentSession session, CancellationToken cancellationToken);
Task DeleteAgentMessageAsync(Int32 poolId, Int64 messageId, Guid sessionId, CancellationToken cancellationToken);
Task DeleteAgentSessionAsync(Int32 poolId, Guid sessionId, CancellationToken cancellationToken);
Task<TaskAgentMessage> GetAgentMessageAsync(Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken);
// job request
Task<TaskAgentJobRequest> GetAgentRequestAsync(int poolId, long requestId, CancellationToken cancellationToken);
Task<TaskAgentJobRequest> RenewAgentRequestAsync(int poolId, long requestId, Guid lockToken, CancellationToken cancellationToken);
Task<TaskAgentJobRequest> FinishAgentRequestAsync(int poolId, long requestId, Guid lockToken, DateTime finishTime, TaskResult result, CancellationToken cancellationToken);
// agent package
Task<List<PackageMetadata>> GetPackagesAsync(string packageType, string platform, int top, CancellationToken cancellationToken);
Task<PackageMetadata> GetPackageAsync(string packageType, string platform, string version, CancellationToken cancellationToken);
// agent update
Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, int agentId, string currentState);
}
public sealed class RunnerServer : RunnerService, IRunnerServer
{
private bool _hasGenericConnection;
private bool _hasMessageConnection;
private bool _hasRequestConnection;
private VssConnection _genericConnection;
private VssConnection _messageConnection;
private VssConnection _requestConnection;
private TaskAgentHttpClient _genericTaskAgentClient;
private TaskAgentHttpClient _messageTaskAgentClient;
private TaskAgentHttpClient _requestTaskAgentClient;
public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials)
{
if (HostContext.RunMode == RunMode.Local)
{
return;
}
var createGenericConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100));
var createMessageConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
var createRequestConnection = EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(60));
await Task.WhenAll(createGenericConnection, createMessageConnection, createRequestConnection);
_genericConnection = await createGenericConnection;
_messageConnection = await createMessageConnection;
_requestConnection = await createRequestConnection;
_genericTaskAgentClient = _genericConnection.GetClient<TaskAgentHttpClient>();
_messageTaskAgentClient = _messageConnection.GetClient<TaskAgentHttpClient>();
_requestTaskAgentClient = _requestConnection.GetClient<TaskAgentHttpClient>();
_hasGenericConnection = true;
_hasMessageConnection = true;
_hasRequestConnection = true;
}
// Refresh connection is best effort. it should never throw exception
public async Task RefreshConnectionAsync(RunnerConnectionType connectionType, TimeSpan timeout)
{
Trace.Info($"Refresh {connectionType} VssConnection to get on a different AFD node.");
VssConnection newConnection = null;
switch (connectionType)
{
case RunnerConnectionType.MessageQueue:
try
{
_hasMessageConnection = false;
newConnection = await EstablishVssConnection(_messageConnection.Uri, _messageConnection.Credentials, timeout);
var client = newConnection.GetClient<TaskAgentHttpClient>();
_messageConnection = newConnection;
_messageTaskAgentClient = client;
}
catch (Exception ex)
{
Trace.Error($"Catch exception during reset {connectionType} connection.");
Trace.Error(ex);
newConnection?.Dispose();
}
finally
{
_hasMessageConnection = true;
}
break;
case RunnerConnectionType.JobRequest:
try
{
_hasRequestConnection = false;
newConnection = await EstablishVssConnection(_requestConnection.Uri, _requestConnection.Credentials, timeout);
var client = newConnection.GetClient<TaskAgentHttpClient>();
_requestConnection = newConnection;
_requestTaskAgentClient = client;
}
catch (Exception ex)
{
Trace.Error($"Catch exception during reset {connectionType} connection.");
Trace.Error(ex);
newConnection?.Dispose();
}
finally
{
_hasRequestConnection = true;
}
break;
case RunnerConnectionType.Generic:
try
{
_hasGenericConnection = false;
newConnection = await EstablishVssConnection(_genericConnection.Uri, _genericConnection.Credentials, timeout);
var client = newConnection.GetClient<TaskAgentHttpClient>();
_genericConnection = newConnection;
_genericTaskAgentClient = client;
}
catch (Exception ex)
{
Trace.Error($"Catch exception during reset {connectionType} connection.");
Trace.Error(ex);
newConnection?.Dispose();
}
finally
{
_hasGenericConnection = true;
}
break;
default:
Trace.Error($"Unexpected connection type: {connectionType}.");
break;
}
}
public void SetConnectionTimeout(RunnerConnectionType connectionType, TimeSpan timeout)
{
Trace.Info($"Set {connectionType} VssConnection's timeout to {timeout.TotalSeconds} seconds.");
switch (connectionType)
{
case RunnerConnectionType.JobRequest:
_requestConnection.Settings.SendTimeout = timeout;
break;
case RunnerConnectionType.MessageQueue:
_messageConnection.Settings.SendTimeout = timeout;
break;
case RunnerConnectionType.Generic:
_genericConnection.Settings.SendTimeout = timeout;
break;
default:
Trace.Error($"Unexpected connection type: {connectionType}.");
break;
}
}
private async Task<VssConnection> EstablishVssConnection(Uri serverUrl, VssCredentials credentials, TimeSpan timeout)
{
Trace.Info($"Establish connection with {timeout.TotalSeconds} seconds timeout.");
int attemptCount = 5;
while (attemptCount-- > 0)
{
var connection = VssUtil.CreateConnection(serverUrl, credentials, timeout: timeout);
try
{
await connection.ConnectAsync();
return connection;
}
catch (Exception ex) when (attemptCount > 0)
{
Trace.Info($"Catch exception during connect. {attemptCount} attempt left.");
Trace.Error(ex);
await HostContext.Delay(TimeSpan.FromMilliseconds(100), CancellationToken.None);
}
}
// should never reach here.
throw new InvalidOperationException(nameof(EstablishVssConnection));
}
private void CheckConnection(RunnerConnectionType connectionType)
{
switch (connectionType)
{
case RunnerConnectionType.Generic:
if (!_hasGenericConnection)
{
throw new InvalidOperationException($"SetConnection {RunnerConnectionType.Generic}");
}
break;
case RunnerConnectionType.JobRequest:
if (!_hasRequestConnection)
{
throw new InvalidOperationException($"SetConnection {RunnerConnectionType.JobRequest}");
}
break;
case RunnerConnectionType.MessageQueue:
if (!_hasMessageConnection)
{
throw new InvalidOperationException($"SetConnection {RunnerConnectionType.MessageQueue}");
}
break;
default:
throw new NotSupportedException(connectionType.ToString());
}
}
//-----------------------------------------------------------------
// Configuration
//-----------------------------------------------------------------
public Task<List<TaskAgentPool>> GetAgentPoolsAsync(string agentPoolName = null, TaskAgentPoolType poolType = TaskAgentPoolType.Automation)
{
CheckConnection(RunnerConnectionType.Generic);
return _genericTaskAgentClient.GetAgentPoolsAsync(agentPoolName, poolType: poolType);
}
public Task<TaskAgent> AddAgentAsync(Int32 agentPoolId, TaskAgent agent)
{
CheckConnection(RunnerConnectionType.Generic);
return _genericTaskAgentClient.AddAgentAsync(agentPoolId, agent);
}
public Task<List<TaskAgent>> GetAgentsAsync(int agentPoolId, string agentName = null)
{
CheckConnection(RunnerConnectionType.Generic);
return _genericTaskAgentClient.GetAgentsAsync(agentPoolId, agentName, false);
}
public Task<TaskAgent> UpdateAgentAsync(int agentPoolId, TaskAgent agent)
{
CheckConnection(RunnerConnectionType.Generic);
return _genericTaskAgentClient.ReplaceAgentAsync(agentPoolId, agent);
}
public Task DeleteAgentAsync(int agentPoolId, int agentId)
{
CheckConnection(RunnerConnectionType.Generic);
return _genericTaskAgentClient.DeleteAgentAsync(agentPoolId, agentId);
}
//-----------------------------------------------------------------
// MessageQueue
//-----------------------------------------------------------------
public Task<TaskAgentSession> CreateAgentSessionAsync(Int32 poolId, TaskAgentSession session, CancellationToken cancellationToken)
{
CheckConnection(RunnerConnectionType.MessageQueue);
return _messageTaskAgentClient.CreateAgentSessionAsync(poolId, session, cancellationToken: cancellationToken);
}
public Task DeleteAgentMessageAsync(Int32 poolId, Int64 messageId, Guid sessionId, CancellationToken cancellationToken)
{
CheckConnection(RunnerConnectionType.MessageQueue);
return _messageTaskAgentClient.DeleteMessageAsync(poolId, messageId, sessionId, cancellationToken: cancellationToken);
}
public Task DeleteAgentSessionAsync(Int32 poolId, Guid sessionId, CancellationToken cancellationToken)
{
CheckConnection(RunnerConnectionType.MessageQueue);
return _messageTaskAgentClient.DeleteAgentSessionAsync(poolId, sessionId, cancellationToken: cancellationToken);
}
public Task<TaskAgentMessage> GetAgentMessageAsync(Int32 poolId, Guid sessionId, Int64? lastMessageId, CancellationToken cancellationToken)
{
CheckConnection(RunnerConnectionType.MessageQueue);
return _messageTaskAgentClient.GetMessageAsync(poolId, sessionId, lastMessageId, cancellationToken: cancellationToken);
}
//-----------------------------------------------------------------
// JobRequest
//-----------------------------------------------------------------
public Task<TaskAgentJobRequest> RenewAgentRequestAsync(int poolId, long requestId, Guid lockToken, CancellationToken cancellationToken = default(CancellationToken))
{
if (HostContext.RunMode == RunMode.Local)
{
return Task.FromResult(JsonUtility.FromString<TaskAgentJobRequest>("{ lockedUntil: \"" + DateTime.Now.Add(TimeSpan.FromMinutes(5)).ToString("u") + "\" }"));
}
CheckConnection(RunnerConnectionType.JobRequest);
return _requestTaskAgentClient.RenewAgentRequestAsync(poolId, requestId, lockToken, cancellationToken: cancellationToken);
}
public Task<TaskAgentJobRequest> FinishAgentRequestAsync(int poolId, long requestId, Guid lockToken, DateTime finishTime, TaskResult result, CancellationToken cancellationToken = default(CancellationToken))
{
if (HostContext.RunMode == RunMode.Local)
{
return Task.FromResult<TaskAgentJobRequest>(null);
}
CheckConnection(RunnerConnectionType.JobRequest);
return _requestTaskAgentClient.FinishAgentRequestAsync(poolId, requestId, lockToken, finishTime, result, cancellationToken: cancellationToken);
}
public Task<TaskAgentJobRequest> GetAgentRequestAsync(int poolId, long requestId, CancellationToken cancellationToken = default(CancellationToken))
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
CheckConnection(RunnerConnectionType.JobRequest);
return _requestTaskAgentClient.GetAgentRequestAsync(poolId, requestId, cancellationToken: cancellationToken);
}
//-----------------------------------------------------------------
// Agent Package
//-----------------------------------------------------------------
public Task<List<PackageMetadata>> GetPackagesAsync(string packageType, string platform, int top, CancellationToken cancellationToken)
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
CheckConnection(RunnerConnectionType.Generic);
return _genericTaskAgentClient.GetPackagesAsync(packageType, platform, top, cancellationToken: cancellationToken);
}
public Task<PackageMetadata> GetPackageAsync(string packageType, string platform, string version, CancellationToken cancellationToken)
{
CheckConnection(RunnerConnectionType.Generic);
return _genericTaskAgentClient.GetPackageAsync(packageType, platform, version, cancellationToken: cancellationToken);
}
public Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, int agentId, string currentState)
{
CheckConnection(RunnerConnectionType.Generic);
return _genericTaskAgentClient.UpdateAgentUpdateStateAsync(agentPoolId, agentId, currentState);
}
}
}

View File

@@ -0,0 +1,39 @@
using System;
namespace GitHub.Runner.Common
{
[AttributeUsage(AttributeTargets.Interface, Inherited = false, AllowMultiple = false)]
public sealed class ServiceLocatorAttribute : Attribute
{
public static readonly string DefaultPropertyName = "Default";
public Type Default { get; set; }
}
public interface IRunnerService
{
void Initialize(IHostContext context);
}
public abstract class RunnerService
{
protected IHostContext HostContext { get; private set; }
protected Tracing Trace { get; private set; }
public string TraceName
{
get
{
return GetType().Name;
}
}
public virtual void Initialize(IHostContext hostContext)
{
HostContext = hostContext;
Trace = HostContext.GetTrace(TraceName);
Trace.Entering();
}
}
}

View File

@@ -0,0 +1,196 @@
using GitHub.Runner.Common.Util;
using System;
using System.Linq;
using System.Net;
using System.IO;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common
{
[ServiceLocator(Default = typeof(RunnerWebProxy))]
public interface IRunnerWebProxy : IRunnerService
{
string ProxyAddress { get; }
string ProxyUsername { get; }
string ProxyPassword { get; }
List<string> ProxyBypassList { get; }
IWebProxy WebProxy { get; }
}
public class RunnerWebProxy : RunnerService, IRunnerWebProxy
{
private readonly List<Regex> _regExBypassList = new List<Regex>();
private readonly List<string> _bypassList = new List<string>();
private RunnerWebProxyCore _runnerWebProxy = new RunnerWebProxyCore();
public string ProxyAddress { get; private set; }
public string ProxyUsername { get; private set; }
public string ProxyPassword { get; private set; }
public List<string> ProxyBypassList => _bypassList;
public IWebProxy WebProxy => _runnerWebProxy;
public override void Initialize(IHostContext context)
{
base.Initialize(context);
LoadProxySetting();
}
// This should only be called from config
public void SetupProxy(string proxyAddress, string proxyUsername, string proxyPassword)
{
ArgUtil.NotNullOrEmpty(proxyAddress, nameof(proxyAddress));
Trace.Info($"Update proxy setting from '{ProxyAddress ?? string.Empty}' to'{proxyAddress}'");
ProxyAddress = proxyAddress;
ProxyUsername = proxyUsername;
ProxyPassword = proxyPassword;
if (string.IsNullOrEmpty(ProxyUsername) || string.IsNullOrEmpty(ProxyPassword))
{
Trace.Info($"Config proxy use DefaultNetworkCredentials.");
}
else
{
Trace.Info($"Config authentication proxy as: {ProxyUsername}.");
}
_runnerWebProxy.Update(ProxyAddress, ProxyUsername, ProxyPassword, ProxyBypassList);
}
// This should only be called from config
public void SaveProxySetting()
{
if (!string.IsNullOrEmpty(ProxyAddress))
{
string proxyConfigFile = HostContext.GetConfigFile(WellKnownConfigFile.Proxy);
IOUtil.DeleteFile(proxyConfigFile);
Trace.Info($"Store proxy configuration to '{proxyConfigFile}' for proxy '{ProxyAddress}'");
File.WriteAllText(proxyConfigFile, ProxyAddress);
File.SetAttributes(proxyConfigFile, File.GetAttributes(proxyConfigFile) | FileAttributes.Hidden);
string proxyCredFile = HostContext.GetConfigFile(WellKnownConfigFile.ProxyCredentials);
IOUtil.DeleteFile(proxyCredFile);
if (!string.IsNullOrEmpty(ProxyUsername) && !string.IsNullOrEmpty(ProxyPassword))
{
string lookupKey = Guid.NewGuid().ToString("D").ToUpperInvariant();
Trace.Info($"Store proxy credential lookup key '{lookupKey}' to '{proxyCredFile}'");
File.WriteAllText(proxyCredFile, lookupKey);
File.SetAttributes(proxyCredFile, File.GetAttributes(proxyCredFile) | FileAttributes.Hidden);
var credStore = HostContext.GetService<IRunnerCredentialStore>();
credStore.Write($"GITHUB_ACTIONS_RUNNER_PROXY_{lookupKey}", ProxyUsername, ProxyPassword);
}
}
else
{
Trace.Info("No proxy configuration exist.");
}
}
// This should only be called from unconfig
public void DeleteProxySetting()
{
string proxyCredFile = HostContext.GetConfigFile(WellKnownConfigFile.ProxyCredentials);
if (File.Exists(proxyCredFile))
{
Trace.Info("Delete proxy credential from credential store.");
string lookupKey = File.ReadAllLines(proxyCredFile).FirstOrDefault();
if (!string.IsNullOrEmpty(lookupKey))
{
var credStore = HostContext.GetService<IRunnerCredentialStore>();
credStore.Delete($"GITHUB_ACTIONS_RUNNER_PROXY_{lookupKey}");
}
Trace.Info($"Delete .proxycredentials file: {proxyCredFile}");
IOUtil.DeleteFile(proxyCredFile);
}
string proxyBypassFile = HostContext.GetConfigFile(WellKnownConfigFile.ProxyBypass);
if (File.Exists(proxyBypassFile))
{
Trace.Info($"Delete .proxybypass file: {proxyBypassFile}");
IOUtil.DeleteFile(proxyBypassFile);
}
string proxyConfigFile = HostContext.GetConfigFile(WellKnownConfigFile.Proxy);
Trace.Info($"Delete .proxy file: {proxyConfigFile}");
IOUtil.DeleteFile(proxyConfigFile);
}
private void LoadProxySetting()
{
string proxyConfigFile = HostContext.GetConfigFile(WellKnownConfigFile.Proxy);
if (File.Exists(proxyConfigFile))
{
// we expect the first line of the file is the proxy url
Trace.Verbose($"Try read proxy setting from file: {proxyConfigFile}.");
ProxyAddress = File.ReadLines(proxyConfigFile).FirstOrDefault() ?? string.Empty;
ProxyAddress = ProxyAddress.Trim();
Trace.Verbose($"{ProxyAddress}");
}
if (!string.IsNullOrEmpty(ProxyAddress) && !Uri.IsWellFormedUriString(ProxyAddress, UriKind.Absolute))
{
Trace.Info($"The proxy url is not a well formed absolute uri string: {ProxyAddress}.");
ProxyAddress = string.Empty;
}
if (!string.IsNullOrEmpty(ProxyAddress))
{
Trace.Info($"Config proxy at: {ProxyAddress}.");
string proxyCredFile = HostContext.GetConfigFile(WellKnownConfigFile.ProxyCredentials);
if (File.Exists(proxyCredFile))
{
string lookupKey = File.ReadAllLines(proxyCredFile).FirstOrDefault();
if (!string.IsNullOrEmpty(lookupKey))
{
var credStore = HostContext.GetService<IRunnerCredentialStore>();
var proxyCred = credStore.Read($"GITHUB_ACTIONS_RUNNER_PROXY_{lookupKey}");
ProxyUsername = proxyCred.UserName;
ProxyPassword = proxyCred.Password;
}
}
if (!string.IsNullOrEmpty(ProxyPassword))
{
HostContext.SecretMasker.AddValue(ProxyPassword);
}
if (string.IsNullOrEmpty(ProxyUsername) || string.IsNullOrEmpty(ProxyPassword))
{
Trace.Info($"Config proxy use DefaultNetworkCredentials.");
}
else
{
Trace.Info($"Config authentication proxy as: {ProxyUsername}.");
}
string proxyBypassFile = HostContext.GetConfigFile(WellKnownConfigFile.ProxyBypass);
if (File.Exists(proxyBypassFile))
{
Trace.Verbose($"Try read proxy bypass list from file: {proxyBypassFile}.");
foreach (string bypass in File.ReadAllLines(proxyBypassFile))
{
if (string.IsNullOrWhiteSpace(bypass))
{
continue;
}
else
{
Trace.Info($"Bypass proxy for: {bypass}.");
ProxyBypassList.Add(bypass.Trim());
}
}
}
_runnerWebProxy.Update(ProxyAddress, ProxyUsername, ProxyPassword, ProxyBypassList);
}
else
{
Trace.Info($"No proxy setting found.");
}
}
}
}

View File

@@ -0,0 +1,96 @@
// Defines the data protocol for reading and writing strings on our stream
using System;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace GitHub.Runner.Common
{
public class StreamString
{
private Stream _ioStream;
private UnicodeEncoding streamEncoding;
public StreamString(Stream ioStream)
{
_ioStream = ioStream;
streamEncoding = new UnicodeEncoding();
}
public async Task<Int32> ReadInt32Async(CancellationToken cancellationToken)
{
byte[] readBytes = new byte[sizeof(Int32)];
int dataread = 0;
while (sizeof(Int32) - dataread > 0 && (!cancellationToken.IsCancellationRequested))
{
Task<int> op = _ioStream.ReadAsync(readBytes, dataread, sizeof(Int32) - dataread, cancellationToken);
int newData = 0;
newData = await op.WithCancellation(cancellationToken);
dataread += newData;
if (0 == newData)
{
await Task.Delay(100, cancellationToken);
}
}
cancellationToken.ThrowIfCancellationRequested();
return BitConverter.ToInt32(readBytes, 0);
}
public async Task WriteInt32Async(Int32 value, CancellationToken cancellationToken)
{
byte[] int32Bytes = BitConverter.GetBytes(value);
Task op = _ioStream.WriteAsync(int32Bytes, 0, sizeof(Int32), cancellationToken);
await op.WithCancellation(cancellationToken);
}
const int MaxStringSize = 50 * 1000000;
public async Task<string> ReadStringAsync(CancellationToken cancellationToken)
{
Int32 len = await ReadInt32Async(cancellationToken);
if (len == 0)
{
return string.Empty;
}
if (len < 0 || len > MaxStringSize)
{
throw new InvalidDataException();
}
byte[] inBuffer = new byte[len];
int dataread = 0;
while (len - dataread > 0 && (!cancellationToken.IsCancellationRequested))
{
Task<int> op = _ioStream.ReadAsync(inBuffer, dataread, len - dataread, cancellationToken);
int newData = 0;
newData = await op.WithCancellation(cancellationToken);
dataread += newData;
if (0 == newData)
{
await Task.Delay(100, cancellationToken);
}
}
return streamEncoding.GetString(inBuffer);
}
public async Task WriteStringAsync(string outString, CancellationToken cancellationToken)
{
byte[] outBuffer = streamEncoding.GetBytes(outString);
Int32 len = outBuffer.Length;
if (len > MaxStringSize)
{
throw new ArgumentOutOfRangeException();
}
await WriteInt32Async(len, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
Task op = _ioStream.WriteAsync(outBuffer, 0, len, cancellationToken);
await op.WithCancellation(cancellationToken);
op = _ioStream.FlushAsync(cancellationToken);
await op.WithCancellation(cancellationToken);
}
}
}

View File

@@ -0,0 +1,198 @@
using System;
using System.Collections.Generic;
namespace GitHub.Runner.Common
{
//
// Abstracts away interactions with the terminal which allows:
// (1) Console writes also go to trace for better context in the trace
// (2) Reroute in tests
//
[ServiceLocator(Default = typeof(Terminal))]
public interface ITerminal : IRunnerService, IDisposable
{
event EventHandler CancelKeyPress;
bool Silent { get; set; }
string ReadLine();
string ReadSecret();
void Write(string message, ConsoleColor? colorCode = null);
void WriteLine();
void WriteLine(string line, ConsoleColor? colorCode = null);
void WriteError(Exception ex);
void WriteError(string line);
void WriteSection(string message);
void WriteSuccessMessage(string message);
}
public sealed class Terminal : RunnerService, ITerminal
{
public bool Silent { get; set; }
public event EventHandler CancelKeyPress;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
Console.CancelKeyPress += Console_CancelKeyPress;
}
private void Console_CancelKeyPress(object sender, ConsoleCancelEventArgs e)
{
e.Cancel = true;
CancelKeyPress?.Invoke(this, e);
}
public string ReadLine()
{
// Read and trace the value.
Trace.Info("READ LINE");
string value = Console.ReadLine();
Trace.Info($"Read value: '{value}'");
return value;
}
// TODO: Consider using SecureString.
public string ReadSecret()
{
Trace.Info("READ SECRET");
var chars = new List<char>();
while (true)
{
ConsoleKeyInfo key = Console.ReadKey(intercept: true);
if (key.Key == ConsoleKey.Enter)
{
Console.WriteLine();
break;
}
else if (key.Key == ConsoleKey.Backspace)
{
if (chars.Count > 0)
{
chars.RemoveAt(chars.Count - 1);
Console.Write("\b \b");
}
}
else if (key.KeyChar > 0)
{
chars.Add(key.KeyChar);
Console.Write("*");
}
}
// Trace whether a value was entered.
string val = new String(chars.ToArray());
if (!string.IsNullOrEmpty(val))
{
HostContext.SecretMasker.AddValue(val);
}
Trace.Info($"Read value: '{val}'");
return val;
}
public void Write(string message, ConsoleColor? colorCode = null)
{
Trace.Info($"WRITE: {message}");
if (!Silent)
{
if(colorCode != null)
{
Console.ForegroundColor = colorCode.Value;
Console.Write(message);
Console.ResetColor();
}
else {
Console.Write(message);
}
}
}
public void WriteLine()
{
WriteLine(string.Empty);
}
// Do not add a format string overload. Terminal messages are user facing and therefore
// should be localized. Use the Loc method in the StringUtil class.
public void WriteLine(string line, ConsoleColor? colorCode = null)
{
Trace.Info($"WRITE LINE: {line}");
if (!Silent)
{
if(colorCode != null)
{
Console.ForegroundColor = colorCode.Value;
Console.WriteLine(line);
Console.ResetColor();
}
else {
Console.WriteLine(line);
}
}
}
public void WriteError(Exception ex)
{
Trace.Error("WRITE ERROR (exception):");
Trace.Error(ex);
if (!Silent)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.Error.WriteLine(ex.Message);
Console.ResetColor();
}
}
// Do not add a format string overload. Terminal messages are user facing and therefore
// should be localized. Use the Loc method in the StringUtil class.
public void WriteError(string line)
{
Trace.Error($"WRITE ERROR: {line}");
if (!Silent)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.Error.WriteLine(line);
Console.ResetColor();
}
}
public void WriteSection(string message)
{
if (!Silent)
{
Console.WriteLine();
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine($"# {message}");
Console.ResetColor();
Console.WriteLine();
}
}
public void WriteSuccessMessage(string message)
{
if (!Silent)
{
Console.ForegroundColor = ConsoleColor.Green;
Console.Write("√ ");
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine(message);
Console.ResetColor();
}
}
private void Dispose(bool disposing)
{
if (disposing)
{
Console.CancelKeyPress -= Console_CancelKeyPress;
}
}
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
}
}

View File

@@ -0,0 +1,65 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using GitHub.Services.Common.Internal;
namespace GitHub.Runner.Common
{
public class ThrottlingEventArgs : EventArgs
{
public ThrottlingEventArgs(TimeSpan delay, DateTime expiration)
{
Delay = delay;
Expiration = expiration;
}
public TimeSpan Delay { get; private set; }
public DateTime Expiration { get; private set; }
}
public interface IThrottlingReporter
{
void ReportThrottling(TimeSpan delay, DateTime expiration);
}
public class ThrottlingReportHandler : DelegatingHandler
{
private IThrottlingReporter _throttlingReporter;
public ThrottlingReportHandler(IThrottlingReporter throttlingReporter)
: base()
{
ArgUtil.NotNull(throttlingReporter, nameof(throttlingReporter));
_throttlingReporter = throttlingReporter;
}
protected async override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
// Call the inner handler.
var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
// Inspect whether response has throttling information
IEnumerable<string> vssRequestDelayed = null;
IEnumerable<string> vssRequestQuotaReset = null;
if (response.Headers.TryGetValues(HttpHeaders.VssRateLimitDelay, out vssRequestDelayed) &&
response.Headers.TryGetValues(HttpHeaders.VssRateLimitReset, out vssRequestQuotaReset) &&
!string.IsNullOrEmpty(vssRequestDelayed.FirstOrDefault()) &&
!string.IsNullOrEmpty(vssRequestQuotaReset.FirstOrDefault()))
{
TimeSpan delay = TimeSpan.FromSeconds(double.Parse(vssRequestDelayed.First()));
int expirationEpoch = int.Parse(vssRequestQuotaReset.First());
DateTime expiration = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(expirationEpoch);
_throttlingReporter.ReportThrottling(delay, expiration);
}
return response;
}
}
}

View File

@@ -0,0 +1,88 @@
using GitHub.Runner.Common.Util;
using System;
using System.Collections.Concurrent;
using System.Diagnostics;
using GitHub.DistributedTask.Logging;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common
{
public interface ITraceManager : IDisposable
{
SourceSwitch Switch { get; }
Tracing this[string name] { get; }
}
public sealed class TraceManager : ITraceManager
{
private readonly ConcurrentDictionary<string, Tracing> _sources = new ConcurrentDictionary<string, Tracing>(StringComparer.OrdinalIgnoreCase);
private readonly HostTraceListener _hostTraceListener;
private TraceSetting _traceSetting;
private ISecretMasker _secretMasker;
public TraceManager(HostTraceListener traceListener, ISecretMasker secretMasker)
: this(traceListener, new TraceSetting(), secretMasker)
{
}
public TraceManager(HostTraceListener traceListener, TraceSetting traceSetting, ISecretMasker secretMasker)
{
// Validate and store params.
ArgUtil.NotNull(traceListener, nameof(traceListener));
ArgUtil.NotNull(traceSetting, nameof(traceSetting));
ArgUtil.NotNull(secretMasker, nameof(secretMasker));
_hostTraceListener = traceListener;
_traceSetting = traceSetting;
_secretMasker = secretMasker;
Switch = new SourceSwitch("GitHubActionsRunnerSwitch")
{
Level = _traceSetting.DefaultTraceLevel.ToSourceLevels()
};
}
public SourceSwitch Switch { get; private set; }
public Tracing this[string name]
{
get
{
return _sources.GetOrAdd(name, key => CreateTraceSource(key));
}
}
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
private void Dispose(bool disposing)
{
if (disposing)
{
foreach (Tracing traceSource in _sources.Values)
{
traceSource.Dispose();
}
_sources.Clear();
}
}
private Tracing CreateTraceSource(string name)
{
SourceSwitch sourceSwitch = Switch;
TraceLevel sourceTraceLevel;
if (_traceSetting.DetailTraceSetting.TryGetValue(name, out sourceTraceLevel))
{
sourceSwitch = new SourceSwitch("GitHubActionsRunnerSubSwitch")
{
Level = sourceTraceLevel.ToSourceLevels()
};
}
return new Tracing(name, _secretMasker, sourceSwitch, _hostTraceListener);
}
}
}

View File

@@ -0,0 +1,92 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.Serialization;
namespace GitHub.Runner.Common
{
[DataContract]
public class TraceSetting
{
public TraceSetting()
{
DefaultTraceLevel = TraceLevel.Info;
#if DEBUG
DefaultTraceLevel = TraceLevel.Verbose;
#endif
string actionsRunnerTrace = Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_TRACE");
if (!string.IsNullOrEmpty(actionsRunnerTrace))
{
DefaultTraceLevel = TraceLevel.Verbose;
}
}
[DataMember(EmitDefaultValue = false)]
public TraceLevel DefaultTraceLevel
{
get;
set;
}
public Dictionary<String, TraceLevel> DetailTraceSetting
{
get
{
if (m_detailTraceSetting == null)
{
m_detailTraceSetting = new Dictionary<String, TraceLevel>(StringComparer.OrdinalIgnoreCase);
}
return m_detailTraceSetting;
}
}
[DataMember(EmitDefaultValue = false, Name = "DetailTraceSetting")]
private Dictionary<String, TraceLevel> m_detailTraceSetting;
}
[DataContract]
public enum TraceLevel
{
[EnumMember]
Off = 0,
[EnumMember]
Critical = 1,
[EnumMember]
Error = 2,
[EnumMember]
Warning = 3,
[EnumMember]
Info = 4,
[EnumMember]
Verbose = 5,
}
public static class TraceLevelExtensions
{
public static SourceLevels ToSourceLevels(this TraceLevel traceLevel)
{
switch (traceLevel)
{
case TraceLevel.Off:
return SourceLevels.Off;
case TraceLevel.Critical:
return SourceLevels.Critical;
case TraceLevel.Error:
return SourceLevels.Error;
case TraceLevel.Warning:
return SourceLevels.Warning;
case TraceLevel.Info:
return SourceLevels.Information;
case TraceLevel.Verbose:
return SourceLevels.Verbose;
default:
return SourceLevels.Information;
}
}
}
}

View File

@@ -0,0 +1,128 @@

using GitHub.Runner.Common.Util;
using Newtonsoft.Json;
using System;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using GitHub.DistributedTask.Logging;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common
{
public sealed class Tracing : ITraceWriter, IDisposable
{
private ISecretMasker _secretMasker;
private TraceSource _traceSource;
public Tracing(string name, ISecretMasker secretMasker, SourceSwitch sourceSwitch, HostTraceListener traceListener)
{
ArgUtil.NotNull(secretMasker, nameof(secretMasker));
_secretMasker = secretMasker;
_traceSource = new TraceSource(name);
_traceSource.Switch = sourceSwitch;
// Remove the default trace listener.
if (_traceSource.Listeners.Count > 0 &&
_traceSource.Listeners[0] is DefaultTraceListener)
{
_traceSource.Listeners.RemoveAt(0);
}
_traceSource.Listeners.Add(traceListener);
}
public void Info(string message)
{
Trace(TraceEventType.Information, message);
}
public void Info(string format, params object[] args)
{
Trace(TraceEventType.Information, StringUtil.Format(format, args));
}
public void Info(object item)
{
string json = JsonConvert.SerializeObject(item, Formatting.Indented);
Trace(TraceEventType.Information, json);
}
public void Error(Exception exception)
{
Trace(TraceEventType.Error, exception.ToString());
}
// Do not remove the non-format overload.
public void Error(string message)
{
Trace(TraceEventType.Error, message);
}
public void Error(string format, params object[] args)
{
Trace(TraceEventType.Error, StringUtil.Format(format, args));
}
// Do not remove the non-format overload.
public void Warning(string message)
{
Trace(TraceEventType.Warning, message);
}
public void Warning(string format, params object[] args)
{
Trace(TraceEventType.Warning, StringUtil.Format(format, args));
}
// Do not remove the non-format overload.
public void Verbose(string message)
{
Trace(TraceEventType.Verbose, message);
}
public void Verbose(string format, params object[] args)
{
Trace(TraceEventType.Verbose, StringUtil.Format(format, args));
}
public void Verbose(object item)
{
string json = JsonConvert.SerializeObject(item, Formatting.Indented);
Trace(TraceEventType.Verbose, json);
}
public void Entering([CallerMemberName] string name = "")
{
Trace(TraceEventType.Verbose, $"Entering {name}");
}
public void Leaving([CallerMemberName] string name = "")
{
Trace(TraceEventType.Verbose, $"Leaving {name}");
}
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
private void Trace(TraceEventType eventType, string message)
{
ArgUtil.NotNull(_traceSource, nameof(_traceSource));
_traceSource.TraceEvent(
eventType: eventType,
id: 0,
message: _secretMasker.MaskSecrets(message));
}
private void Dispose(bool disposing)
{
if (disposing)
{
_traceSource.Flush();
_traceSource.Close();
}
}
}
}

View File

@@ -0,0 +1,18 @@
namespace GitHub.Runner.Common.Util
{
using System;
public static class EnumUtil
{
public static T? TryParse<T>(string value) where T: struct
{
T val;
if (Enum.TryParse(value ?? string.Empty, ignoreCase: true, result: out val))
{
return val;
}
return null;
}
}
}

View File

@@ -0,0 +1,28 @@
using System;
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common.Util
{
public static class PlanUtil
{
public static PlanFeatures GetFeatures(TaskOrchestrationPlanReference plan)
{
ArgUtil.NotNull(plan, nameof(plan));
PlanFeatures features = PlanFeatures.None;
if (plan.Version >= 8)
{
features |= PlanFeatures.JobCompletedPlanEvent;
}
return features;
}
}
[Flags]
public enum PlanFeatures
{
None = 0,
JobCompletedPlanEvent = 1,
}
}

View File

@@ -0,0 +1,79 @@
using GitHub.DistributedTask.WebApi;
using System;
namespace GitHub.Runner.Common.Util
{
public static class TaskResultUtil
{
private static readonly int _returnCodeOffset = 100;
public static bool IsValidReturnCode(int returnCode)
{
int resultInt = returnCode - _returnCodeOffset;
return Enum.IsDefined(typeof(TaskResult), resultInt);
}
public static int TranslateToReturnCode(TaskResult result)
{
return _returnCodeOffset + (int)result;
}
public static TaskResult TranslateFromReturnCode(int returnCode)
{
int resultInt = returnCode - _returnCodeOffset;
if (Enum.IsDefined(typeof(TaskResult), resultInt))
{
return (TaskResult)resultInt;
}
else
{
return TaskResult.Failed;
}
}
// Merge 2 TaskResults get the worst result.
// Succeeded -> Failed/Canceled/Skipped/Abandoned
// Failed -> Failed/Canceled
// Canceled -> Canceled
// Skipped -> Skipped
// Abandoned -> Abandoned
public static TaskResult MergeTaskResults(TaskResult? currentResult, TaskResult comingResult)
{
if (currentResult == null)
{
return comingResult;
}
// current result is Canceled/Skip/Abandoned
if (currentResult > TaskResult.Failed)
{
return currentResult.Value;
}
// comming result is bad than current result
if (comingResult >= currentResult)
{
return comingResult;
}
return currentResult.Value;
}
public static ActionResult ToActionResult(this TaskResult result)
{
switch (result)
{
case TaskResult.Succeeded:
return ActionResult.Success;
case TaskResult.Failed:
return ActionResult.Failure;
case TaskResult.Canceled:
return ActionResult.Cancelled;
case TaskResult.Skipped:
return ActionResult.Skipped;
default:
throw new NotSupportedException(result.ToString());
}
}
}
}

View File

@@ -0,0 +1,79 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common.Util
{
[ServiceLocator(Default = typeof(UnixUtil))]
public interface IUnixUtil : IRunnerService
{
Task ExecAsync(string workingDirectory, string toolName, string argLine);
Task ChmodAsync(string mode, string file);
Task ChownAsync(string owner, string group, string file);
}
public sealed class UnixUtil : RunnerService, IUnixUtil
{
private ITerminal _term;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_term = hostContext.GetService<ITerminal>();
}
public async Task ChmodAsync(string mode, string file)
{
Trace.Entering();
await ExecAsync(HostContext.GetDirectory(WellKnownDirectory.Root), "chmod", $"{mode} \"{file}\"");
}
public async Task ChownAsync(string owner, string group, string file)
{
Trace.Entering();
await ExecAsync(HostContext.GetDirectory(WellKnownDirectory.Root), "chown", $"{owner}:{group} \"{file}\"");
}
public async Task ExecAsync(string workingDirectory, string toolName, string argLine)
{
Trace.Entering();
string toolPath = WhichUtil.Which(toolName, trace: Trace);
Trace.Info($"Running {toolPath} {argLine}");
var processInvoker = HostContext.CreateService<IProcessInvoker>();
processInvoker.OutputDataReceived += OnOutputDataReceived;
processInvoker.ErrorDataReceived += OnErrorDataReceived;
try
{
using (var cs = new CancellationTokenSource(TimeSpan.FromSeconds(45)))
{
await processInvoker.ExecuteAsync(workingDirectory, toolPath, argLine, null, true, cs.Token);
}
}
finally
{
processInvoker.OutputDataReceived -= OnOutputDataReceived;
processInvoker.ErrorDataReceived -= OnErrorDataReceived;
}
}
private void OnOutputDataReceived(object sender, ProcessDataReceivedEventArgs e)
{
if (!string.IsNullOrEmpty(e.Data))
{
_term.WriteLine(e.Data);
}
}
private void OnErrorDataReceived(object sender, ProcessDataReceivedEventArgs e)
{
if (!string.IsNullOrEmpty(e.Data))
{
_term.WriteLine(e.Data);
}
}
}
}

View File

@@ -0,0 +1,63 @@
using System;
using System.Collections.Generic;
using System.Linq;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common.Util
{
public static class VarUtil
{
public static StringComparer EnvironmentVariableKeyComparer
{
get
{
switch (Constants.Runner.Platform)
{
case Constants.OSPlatform.Linux:
case Constants.OSPlatform.OSX:
return StringComparer.Ordinal;
case Constants.OSPlatform.Windows:
return StringComparer.OrdinalIgnoreCase;
default:
throw new NotSupportedException(); // Should never reach here.
}
}
}
public static string OS
{
get
{
switch (Constants.Runner.Platform)
{
case Constants.OSPlatform.Linux:
return "Linux";
case Constants.OSPlatform.OSX:
return "macOS";
case Constants.OSPlatform.Windows:
return "Windows";
default:
throw new NotSupportedException(); // Should never reach here.
}
}
}
public static string OSArchitecture
{
get
{
switch (Constants.Runner.PlatformArchitecture)
{
case Constants.Architecture.X86:
return "X86";
case Constants.Architecture.X64:
return "X64";
case Constants.Architecture.Arm:
return "ARM";
default:
throw new NotSupportedException(); // Should never reach here.
}
}
}
}
}

View File

@@ -0,0 +1,493 @@
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Listener.Configuration;
using GitHub.Runner.Common.Util;
using System;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Services.WebApi;
using Pipelines = GitHub.DistributedTask.Pipelines;
using System.IO;
using System.Reflection;
using System.Runtime.CompilerServices;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Listener
{
[ServiceLocator(Default = typeof(Runner))]
public interface IRunner : IRunnerService
{
Task<int> ExecuteCommand(CommandSettings command);
}
public sealed class Runner : RunnerService, IRunner
{
private IMessageListener _listener;
private ITerminal _term;
private bool _inConfigStage;
private ManualResetEvent _completedCommand = new ManualResetEvent(false);
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_term = HostContext.GetService<ITerminal>();
}
public async Task<int> ExecuteCommand(CommandSettings command)
{
try
{
var runnerWebProxy = HostContext.GetService<IRunnerWebProxy>();
var runnerCertManager = HostContext.GetService<IRunnerCertificateManager>();
VssUtil.InitializeVssClientSettings(HostContext.UserAgent, runnerWebProxy.WebProxy, runnerCertManager.VssClientCertificateManager);
_inConfigStage = true;
_completedCommand.Reset();
_term.CancelKeyPress += CtrlCHandler;
//register a SIGTERM handler
HostContext.Unloading += Runner_Unloading;
// TODO Unit test to cover this logic
Trace.Info(nameof(ExecuteCommand));
var configManager = HostContext.GetService<IConfigurationManager>();
// command is not required, if no command it just starts if configured
// TODO: Invalid config prints usage
if (command.Help)
{
PrintUsage(command);
return Constants.Runner.ReturnCode.Success;
}
if (command.Version)
{
_term.WriteLine(BuildConstants.RunnerPackage.Version);
return Constants.Runner.ReturnCode.Success;
}
if (command.Commit)
{
_term.WriteLine(BuildConstants.Source.CommitHash);
return Constants.Runner.ReturnCode.Success;
}
// Configure runner prompt for args if not supplied
// Unattended configure mode will not prompt for args if not supplied and error on any missing or invalid value.
if (command.Configure)
{
try
{
await configManager.ConfigureAsync(command);
return Constants.Runner.ReturnCode.Success;
}
catch (Exception ex)
{
Trace.Error(ex);
_term.WriteError(ex.Message);
return Constants.Runner.ReturnCode.TerminatedError;
}
}
// remove config files, remove service, and exit
if (command.Remove)
{
try
{
await configManager.UnconfigureAsync(command);
return Constants.Runner.ReturnCode.Success;
}
catch (Exception ex)
{
Trace.Error(ex);
_term.WriteError(ex.Message);
return Constants.Runner.ReturnCode.TerminatedError;
}
}
_inConfigStage = false;
// warmup runner process (JIT/CLR)
// In scenarios where the runner is single use (used and then thrown away), the system provisioning the runner can call `Runner.Listener --warmup` before the machine is made available to the pool for use.
// this will optimizes the runner process startup time.
if (command.Warmup)
{
var binDir = HostContext.GetDirectory(WellKnownDirectory.Bin);
foreach (var assemblyFile in Directory.EnumerateFiles(binDir, "*.dll"))
{
try
{
Trace.Info($"Load assembly: {assemblyFile}.");
var assembly = Assembly.LoadFrom(assemblyFile);
var types = assembly.GetTypes();
foreach (Type loadedType in types)
{
try
{
Trace.Info($"Load methods: {loadedType.FullName}.");
var methods = loadedType.GetMethods(BindingFlags.DeclaredOnly | BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static);
foreach (var method in methods)
{
if (!method.IsAbstract && !method.ContainsGenericParameters)
{
Trace.Verbose($"Prepare method: {method.Name}.");
RuntimeHelpers.PrepareMethod(method.MethodHandle);
}
}
}
catch (Exception ex)
{
Trace.Error(ex);
}
}
}
catch (Exception ex)
{
Trace.Error(ex);
}
}
return Constants.Runner.ReturnCode.Success;
}
RunnerSettings settings = configManager.LoadSettings();
var store = HostContext.GetService<IConfigurationStore>();
bool configuredAsService = store.IsServiceConfigured();
// Run runner
if (command.Run) // this line is current break machine provisioner.
{
// Error if runner not configured.
if (!configManager.IsConfigured())
{
_term.WriteError("Runner is not configured.");
PrintUsage(command);
return Constants.Runner.ReturnCode.TerminatedError;
}
Trace.Verbose($"Configured as service: '{configuredAsService}'");
//Get the startup type of the runner i.e., autostartup, service, manual
StartupType startType;
var startupTypeAsString = command.GetStartupType();
if (string.IsNullOrEmpty(startupTypeAsString) && configuredAsService)
{
// We need try our best to make the startup type accurate
// The problem is coming from runner autoupgrade, which result an old version service host binary but a newer version runner binary
// At that time the servicehost won't pass --startuptype to Runner.Listener while the runner is actually running as service.
// We will guess the startup type only when the runner is configured as service and the guess will based on whether STDOUT/STDERR/STDIN been redirect or not
Trace.Info($"Try determine runner startup type base on console redirects.");
startType = (Console.IsErrorRedirected && Console.IsInputRedirected && Console.IsOutputRedirected) ? StartupType.Service : StartupType.Manual;
}
else
{
if (!Enum.TryParse(startupTypeAsString, true, out startType))
{
Trace.Info($"Could not parse the argument value '{startupTypeAsString}' for StartupType. Defaulting to {StartupType.Manual}");
startType = StartupType.Manual;
}
}
#if !OS_WINDOWS
// Fix the work folder setting on Linux
if (settings.WorkFolder.Contains("vsts", StringComparison.OrdinalIgnoreCase))
{
var workFolder = "/runner/work";
var unix = HostContext.GetService<IUnixUtil>();
// create new work folder /runner/work
await unix.ExecAsync(HostContext.GetDirectory(WellKnownDirectory.Root), "sh", $"-c \"sudo mkdir -p {workFolder}\"");
// fix permission
await unix.ExecAsync(HostContext.GetDirectory(WellKnownDirectory.Root), "sh", $"-c \"sudo chown -R $USER {workFolder}\"");
// update settings
settings.WorkFolder = workFolder;
store.SaveSettings(settings);
}
#endif
Trace.Info($"Set runner startup type - {startType}");
HostContext.StartupType = startType;
// Run the runner interactively or as service
return await RunAsync(settings, command.RunOnce);
}
else
{
PrintUsage(command);
return Constants.Runner.ReturnCode.Success;
}
}
finally
{
_term.CancelKeyPress -= CtrlCHandler;
HostContext.Unloading -= Runner_Unloading;
_completedCommand.Set();
}
}
private void Runner_Unloading(object sender, EventArgs e)
{
if ((!_inConfigStage) && (!HostContext.RunnerShutdownToken.IsCancellationRequested))
{
HostContext.ShutdownRunner(ShutdownReason.UserCancelled);
_completedCommand.WaitOne(Constants.Runner.ExitOnUnloadTimeout);
}
}
private void CtrlCHandler(object sender, EventArgs e)
{
_term.WriteLine("Exiting...");
if (_inConfigStage)
{
HostContext.Dispose();
Environment.Exit(Constants.Runner.ReturnCode.TerminatedError);
}
else
{
ConsoleCancelEventArgs cancelEvent = e as ConsoleCancelEventArgs;
if (cancelEvent != null && HostContext.GetService<IConfigurationStore>().IsServiceConfigured())
{
ShutdownReason reason;
if (cancelEvent.SpecialKey == ConsoleSpecialKey.ControlBreak)
{
Trace.Info("Received Ctrl-Break signal from runner service host, this indicate the operating system is shutting down.");
reason = ShutdownReason.OperatingSystemShutdown;
}
else
{
Trace.Info("Received Ctrl-C signal, stop Runner.Listener and Runner.Worker.");
reason = ShutdownReason.UserCancelled;
}
HostContext.ShutdownRunner(reason);
}
else
{
HostContext.ShutdownRunner(ShutdownReason.UserCancelled);
}
}
}
//create worker manager, create message listener and start listening to the queue
private async Task<int> RunAsync(RunnerSettings settings, bool runOnce = false)
{
try
{
Trace.Info(nameof(RunAsync));
_listener = HostContext.GetService<IMessageListener>();
if (!await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken))
{
return Constants.Runner.ReturnCode.TerminatedError;
}
HostContext.WritePerfCounter("SessionCreated");
_term.WriteLine($"{DateTime.UtcNow:u}: Listening for Jobs");
IJobDispatcher jobDispatcher = null;
CancellationTokenSource messageQueueLoopTokenSource = CancellationTokenSource.CreateLinkedTokenSource(HostContext.RunnerShutdownToken);
try
{
var notification = HostContext.GetService<IJobNotification>();
if (!String.IsNullOrEmpty(settings.NotificationSocketAddress))
{
notification.StartClient(settings.NotificationSocketAddress, settings.MonitorSocketAddress);
}
else
{
notification.StartClient(settings.NotificationPipeName, settings.MonitorSocketAddress, HostContext.RunnerShutdownToken);
}
bool autoUpdateInProgress = false;
Task<bool> selfUpdateTask = null;
bool runOnceJobReceived = false;
jobDispatcher = HostContext.CreateService<IJobDispatcher>();
while (!HostContext.RunnerShutdownToken.IsCancellationRequested)
{
TaskAgentMessage message = null;
bool skipMessageDeletion = false;
try
{
Task<TaskAgentMessage> getNextMessage = _listener.GetNextMessageAsync(messageQueueLoopTokenSource.Token);
if (autoUpdateInProgress)
{
Trace.Verbose("Auto update task running at backend, waiting for getNextMessage or selfUpdateTask to finish.");
Task completeTask = await Task.WhenAny(getNextMessage, selfUpdateTask);
if (completeTask == selfUpdateTask)
{
autoUpdateInProgress = false;
if (await selfUpdateTask)
{
Trace.Info("Auto update task finished at backend, an runner update is ready to apply exit the current runner instance.");
Trace.Info("Stop message queue looping.");
messageQueueLoopTokenSource.Cancel();
try
{
await getNextMessage;
}
catch (Exception ex)
{
Trace.Info($"Ignore any exception after cancel message loop. {ex}");
}
if (runOnce)
{
return Constants.Runner.ReturnCode.RunOnceRunnerUpdating;
}
else
{
return Constants.Runner.ReturnCode.RunnerUpdating;
}
}
else
{
Trace.Info("Auto update task finished at backend, there is no available runner update needs to apply, continue message queue looping.");
}
}
}
if (runOnceJobReceived)
{
Trace.Verbose("One time used runner has start running its job, waiting for getNextMessage or the job to finish.");
Task completeTask = await Task.WhenAny(getNextMessage, jobDispatcher.RunOnceJobCompleted.Task);
if (completeTask == jobDispatcher.RunOnceJobCompleted.Task)
{
Trace.Info("Job has finished at backend, the runner will exit since it is running under onetime use mode.");
Trace.Info("Stop message queue looping.");
messageQueueLoopTokenSource.Cancel();
try
{
await getNextMessage;
}
catch (Exception ex)
{
Trace.Info($"Ignore any exception after cancel message loop. {ex}");
}
return Constants.Runner.ReturnCode.Success;
}
}
message = await getNextMessage; //get next message
HostContext.WritePerfCounter($"MessageReceived_{message.MessageType}");
if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase))
{
if (autoUpdateInProgress == false)
{
autoUpdateInProgress = true;
var runnerUpdateMessage = JsonUtility.FromString<AgentRefreshMessage>(message.Body);
var selfUpdater = HostContext.GetService<ISelfUpdater>();
selfUpdateTask = selfUpdater.SelfUpdate(runnerUpdateMessage, jobDispatcher, !runOnce && HostContext.StartupType != StartupType.Service, HostContext.RunnerShutdownToken);
Trace.Info("Refresh message received, kick-off selfupdate background process.");
}
else
{
Trace.Info("Refresh message received, skip autoupdate since a previous autoupdate is already running.");
}
}
else if (string.Equals(message.MessageType, JobRequestMessageTypes.PipelineAgentJobRequest, StringComparison.OrdinalIgnoreCase))
{
if (autoUpdateInProgress || runOnceJobReceived)
{
skipMessageDeletion = true;
Trace.Info($"Skip message deletion for job request message '{message.MessageId}'.");
}
else
{
var jobMessage = StringUtil.ConvertFromJson<Pipelines.AgentJobRequestMessage>(message.Body);
jobDispatcher.Run(jobMessage, runOnce);
if (runOnce)
{
Trace.Info("One time used runner received job message.");
runOnceJobReceived = true;
}
}
}
else if (string.Equals(message.MessageType, JobCancelMessage.MessageType, StringComparison.OrdinalIgnoreCase))
{
var cancelJobMessage = JsonUtility.FromString<JobCancelMessage>(message.Body);
bool jobCancelled = jobDispatcher.Cancel(cancelJobMessage);
skipMessageDeletion = (autoUpdateInProgress || runOnceJobReceived) && !jobCancelled;
if (skipMessageDeletion)
{
Trace.Info($"Skip message deletion for cancellation message '{message.MessageId}'.");
}
}
else
{
Trace.Error($"Received message {message.MessageId} with unsupported message type {message.MessageType}.");
}
}
finally
{
if (!skipMessageDeletion && message != null)
{
try
{
await _listener.DeleteMessageAsync(message);
}
catch (Exception ex)
{
Trace.Error($"Catch exception during delete message from message queue. message id: {message.MessageId}");
Trace.Error(ex);
}
finally
{
message = null;
}
}
}
}
}
finally
{
if (jobDispatcher != null)
{
await jobDispatcher.ShutdownAsync();
}
//TODO: make sure we don't mask more important exception
await _listener.DeleteSessionAsync();
messageQueueLoopTokenSource.Dispose();
}
}
catch (TaskAgentAccessTokenExpiredException)
{
Trace.Info("Agent OAuth token has been revoked. Shutting down.");
}
return Constants.Runner.ReturnCode.Success;
}
private void PrintUsage(CommandSettings command)
{
string separator;
string ext;
#if OS_WINDOWS
separator = "\\";
ext = "cmd";
#else
separator = "/";
ext = "sh";
#endif
_term.WriteLine($@"
Commands:,
.{separator}config.{ext} Configures the runner
.{separator}config.{ext} remove Unconfigures the runner
.{separator}run.{ext} Runs the runner interactively. Does not require any options.
Options:
--version Prints the runner version
--commit Prints the runner commit
--help Prints the help for each command
");
}
}
}

View File

@@ -0,0 +1,467 @@
using GitHub.Runner.Listener.Configuration;
using GitHub.Runner.Common.Util;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using GitHub.DistributedTask.Logging;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Listener
{
public sealed class CommandSettings
{
private readonly Dictionary<string, string> _envArgs = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
private readonly CommandLineParser _parser;
private readonly IPromptManager _promptManager;
private readonly Tracing _trace;
private readonly string[] validCommands =
{
Constants.Runner.CommandLine.Commands.Configure,
Constants.Runner.CommandLine.Commands.Remove,
Constants.Runner.CommandLine.Commands.Run,
Constants.Runner.CommandLine.Commands.Warmup,
};
private readonly string[] validFlags =
{
Constants.Runner.CommandLine.Flags.Commit,
#if OS_WINDOWS
Constants.Runner.CommandLine.Flags.GitUseSChannel,
#endif
Constants.Runner.CommandLine.Flags.Help,
Constants.Runner.CommandLine.Flags.Replace,
Constants.Runner.CommandLine.Flags.RunAsService,
Constants.Runner.CommandLine.Flags.Once,
Constants.Runner.CommandLine.Flags.SslSkipCertValidation,
Constants.Runner.CommandLine.Flags.Unattended,
Constants.Runner.CommandLine.Flags.Version
};
private readonly string[] validArgs =
{
Constants.Runner.CommandLine.Args.Agent,
Constants.Runner.CommandLine.Args.Auth,
Constants.Runner.CommandLine.Args.MonitorSocketAddress,
Constants.Runner.CommandLine.Args.NotificationPipeName,
Constants.Runner.CommandLine.Args.Password,
Constants.Runner.CommandLine.Args.Pool,
Constants.Runner.CommandLine.Args.ProxyPassword,
Constants.Runner.CommandLine.Args.ProxyUrl,
Constants.Runner.CommandLine.Args.ProxyUserName,
Constants.Runner.CommandLine.Args.SslCACert,
Constants.Runner.CommandLine.Args.SslClientCert,
Constants.Runner.CommandLine.Args.SslClientCertKey,
Constants.Runner.CommandLine.Args.SslClientCertArchive,
Constants.Runner.CommandLine.Args.SslClientCertPassword,
Constants.Runner.CommandLine.Args.StartupType,
Constants.Runner.CommandLine.Args.Token,
Constants.Runner.CommandLine.Args.Url,
Constants.Runner.CommandLine.Args.UserName,
Constants.Runner.CommandLine.Args.WindowsLogonAccount,
Constants.Runner.CommandLine.Args.WindowsLogonPassword,
Constants.Runner.CommandLine.Args.Work
};
// Commands.
public bool Configure => TestCommand(Constants.Runner.CommandLine.Commands.Configure);
public bool Remove => TestCommand(Constants.Runner.CommandLine.Commands.Remove);
public bool Run => TestCommand(Constants.Runner.CommandLine.Commands.Run);
public bool Warmup => TestCommand(Constants.Runner.CommandLine.Commands.Warmup);
// Flags.
public bool Commit => TestFlag(Constants.Runner.CommandLine.Flags.Commit);
public bool Help => TestFlag(Constants.Runner.CommandLine.Flags.Help);
public bool Unattended => TestFlag(Constants.Runner.CommandLine.Flags.Unattended);
public bool Version => TestFlag(Constants.Runner.CommandLine.Flags.Version);
#if OS_WINDOWS
public bool GitUseSChannel => TestFlag(Constants.Runner.CommandLine.Flags.GitUseSChannel);
#endif
public bool RunOnce => TestFlag(Constants.Runner.CommandLine.Flags.Once);
// Constructor.
public CommandSettings(IHostContext context, string[] args)
{
ArgUtil.NotNull(context, nameof(context));
_promptManager = context.GetService<IPromptManager>();
_trace = context.GetTrace(nameof(CommandSettings));
// Parse the command line args.
_parser = new CommandLineParser(
hostContext: context,
secretArgNames: Constants.Runner.CommandLine.Args.Secrets);
_parser.Parse(args);
// Store and remove any args passed via environment variables.
IDictionary environment = Environment.GetEnvironmentVariables();
string envPrefix = "ACTIONS_RUNNER_INPUT_";
foreach (DictionaryEntry entry in environment)
{
// Test if starts with ACTIONS_RUNNER_INPUT_.
string fullKey = entry.Key as string ?? string.Empty;
if (fullKey.StartsWith(envPrefix, StringComparison.OrdinalIgnoreCase))
{
string val = (entry.Value as string ?? string.Empty).Trim();
if (!string.IsNullOrEmpty(val))
{
// Extract the name.
string name = fullKey.Substring(envPrefix.Length);
// Mask secrets.
bool secret = Constants.Runner.CommandLine.Args.Secrets.Any(x => string.Equals(x, name, StringComparison.OrdinalIgnoreCase));
if (secret)
{
context.SecretMasker.AddValue(val);
}
// Store the value.
_envArgs[name] = val;
}
// Remove from the environment block.
_trace.Info($"Removing env var: '{fullKey}'");
Environment.SetEnvironmentVariable(fullKey, null);
}
}
}
// Validate commandline parser result
public List<string> Validate()
{
List<string> unknowns = new List<string>();
// detect unknown commands
unknowns.AddRange(_parser.Commands.Where(x => !validCommands.Contains(x, StringComparer.OrdinalIgnoreCase)));
// detect unknown flags
unknowns.AddRange(_parser.Flags.Where(x => !validFlags.Contains(x, StringComparer.OrdinalIgnoreCase)));
// detect unknown args
unknowns.AddRange(_parser.Args.Keys.Where(x => !validArgs.Contains(x, StringComparer.OrdinalIgnoreCase)));
return unknowns;
}
//
// Interactive flags.
//
public bool GetReplace()
{
return TestFlagOrPrompt(
name: Constants.Runner.CommandLine.Flags.Replace,
description: "Would you like to replace the existing runner? (Y/N)",
defaultValue: false);
}
public bool GetRunAsService()
{
return TestFlagOrPrompt(
name: Constants.Runner.CommandLine.Flags.RunAsService,
description: "Would you like to run the runner as service? (Y/N)",
defaultValue: false);
}
public bool GetAutoLaunchBrowser()
{
return TestFlagOrPrompt(
name: Constants.Runner.CommandLine.Flags.LaunchBrowser,
description: "Would you like to launch your browser for AAD Device Code Flow? (Y/N)",
defaultValue: true);
}
//
// Args.
//
public string GetAgentName()
{
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.Agent,
description: "Enter the name of runner:",
defaultValue: Environment.MachineName ?? "myagent",
validator: Validators.NonEmptyValidator);
}
public string GetAuth(string defaultValue)
{
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.Auth,
description: "How would you like to authenticate?",
defaultValue: defaultValue,
validator: Validators.AuthSchemeValidator);
}
public string GetPassword()
{
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.Password,
description: "What is your GitHub password?",
defaultValue: string.Empty,
validator: Validators.NonEmptyValidator);
}
public string GetPool()
{
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.Pool,
description: "Enter the name of your runner pool:",
defaultValue: "default",
validator: Validators.NonEmptyValidator);
}
public string GetToken()
{
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.Token,
description: "Enter your personal access token:",
defaultValue: string.Empty,
validator: Validators.NonEmptyValidator);
}
public string GetRunnerRegisterToken()
{
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.Token,
description: "Enter runner register token:",
defaultValue: string.Empty,
validator: Validators.NonEmptyValidator);
}
public string GetUrl(bool suppressPromptIfEmpty = false)
{
// Note, GetArg does not consume the arg (like GetArgOrPrompt does).
if (suppressPromptIfEmpty &&
string.IsNullOrEmpty(GetArg(Constants.Runner.CommandLine.Args.Url)))
{
return string.Empty;
}
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.Url,
description: "What is the URL of your repository?",
defaultValue: string.Empty,
validator: Validators.ServerUrlValidator);
}
public string GetUserName()
{
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.UserName,
description: "What is your GitHub username?",
defaultValue: string.Empty,
validator: Validators.NonEmptyValidator);
}
public string GetWindowsLogonAccount(string defaultValue, string descriptionMsg)
{
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.WindowsLogonAccount,
description: descriptionMsg,
defaultValue: defaultValue,
validator: Validators.NTAccountValidator);
}
public string GetWindowsLogonPassword(string accountName)
{
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.WindowsLogonPassword,
description: $"Password for the account {accountName}",
defaultValue: string.Empty,
validator: Validators.NonEmptyValidator);
}
public string GetWork()
{
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.Work,
description: "Enter name of work folder:",
defaultValue: Constants.Path.WorkDirectory,
validator: Validators.NonEmptyValidator);
}
public string GetMonitorSocketAddress()
{
return GetArg(Constants.Runner.CommandLine.Args.MonitorSocketAddress);
}
public string GetNotificationPipeName()
{
return GetArg(Constants.Runner.CommandLine.Args.NotificationPipeName);
}
public string GetNotificationSocketAddress()
{
return GetArg(Constants.Runner.CommandLine.Args.NotificationSocketAddress);
}
// This is used to find out the source from where the Runner.Listener.exe was launched at the time of run
public string GetStartupType()
{
return GetArg(Constants.Runner.CommandLine.Args.StartupType);
}
public string GetProxyUrl()
{
return GetArg(Constants.Runner.CommandLine.Args.ProxyUrl);
}
public string GetProxyUserName()
{
return GetArg(Constants.Runner.CommandLine.Args.ProxyUserName);
}
public string GetProxyPassword()
{
return GetArg(Constants.Runner.CommandLine.Args.ProxyPassword);
}
public bool GetSkipCertificateValidation()
{
return TestFlag(Constants.Runner.CommandLine.Flags.SslSkipCertValidation);
}
public string GetCACertificate()
{
return GetArg(Constants.Runner.CommandLine.Args.SslCACert);
}
public string GetClientCertificate()
{
return GetArg(Constants.Runner.CommandLine.Args.SslClientCert);
}
public string GetClientCertificatePrivateKey()
{
return GetArg(Constants.Runner.CommandLine.Args.SslClientCertKey);
}
public string GetClientCertificateArchrive()
{
return GetArg(Constants.Runner.CommandLine.Args.SslClientCertArchive);
}
public string GetClientCertificatePassword()
{
return GetArg(Constants.Runner.CommandLine.Args.SslClientCertPassword);
}
//
// Private helpers.
//
private string GetArg(string name)
{
string result;
if (!_parser.Args.TryGetValue(name, out result))
{
result = GetEnvArg(name);
}
return result;
}
private void RemoveArg(string name)
{
if (_parser.Args.ContainsKey(name))
{
_parser.Args.Remove(name);
}
if (_envArgs.ContainsKey(name))
{
_envArgs.Remove(name);
}
}
private string GetArgOrPrompt(
string name,
string description,
string defaultValue,
Func<string, bool> validator)
{
// Check for the arg in the command line parser.
ArgUtil.NotNull(validator, nameof(validator));
string result = GetArg(name);
// Return the arg if it is not empty and is valid.
_trace.Info($"Arg '{name}': '{result}'");
if (!string.IsNullOrEmpty(result))
{
// After read the arg from input commandline args, remove it from Arg dictionary,
// This will help if bad arg value passed through CommandLine arg, when ConfigurationManager ask CommandSetting the second time,
// It will prompt for input instead of continue use the bad input.
_trace.Info($"Remove {name} from Arg dictionary.");
RemoveArg(name);
if (validator(result))
{
return result;
}
_trace.Info("Arg is invalid.");
}
// Otherwise prompt for the arg.
return _promptManager.ReadValue(
argName: name,
description: description,
secret: Constants.Runner.CommandLine.Args.Secrets.Any(x => string.Equals(x, name, StringComparison.OrdinalIgnoreCase)),
defaultValue: defaultValue,
validator: validator,
unattended: Unattended);
}
private string GetEnvArg(string name)
{
string val;
if (_envArgs.TryGetValue(name, out val) && !string.IsNullOrEmpty(val))
{
_trace.Info($"Env arg '{name}': '{val}'");
return val;
}
return null;
}
private bool TestCommand(string name)
{
bool result = _parser.IsCommand(name);
_trace.Info($"Command '{name}': '{result}'");
return result;
}
private bool TestFlag(string name)
{
bool result = _parser.Flags.Contains(name);
if (!result)
{
string envStr = GetEnvArg(name);
if (!bool.TryParse(envStr, out result))
{
result = false;
}
}
_trace.Info($"Flag '{name}': '{result}'");
return result;
}
private bool TestFlagOrPrompt(
string name,
string description,
bool defaultValue)
{
bool result = TestFlag(name);
if (!result)
{
result = _promptManager.ReadBool(
argName: name,
description: description,
defaultValue: defaultValue,
unattended: Unattended);
}
return result;
}
}
}

View File

@@ -0,0 +1,667 @@
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common.Capabilities;
using GitHub.Runner.Common.Util;
using GitHub.Services.Common;
using GitHub.Services.OAuth;
using GitHub.Services.WebApi;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Security.Principal;
using System.Threading;
using System.Threading.Tasks;
using System.Runtime.InteropServices;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
namespace GitHub.Runner.Listener.Configuration
{
[ServiceLocator(Default = typeof(ConfigurationManager))]
public interface IConfigurationManager : IRunnerService
{
bool IsConfigured();
Task ConfigureAsync(CommandSettings command);
Task UnconfigureAsync(CommandSettings command);
RunnerSettings LoadSettings();
}
public sealed class ConfigurationManager : RunnerService, IConfigurationManager
{
private IConfigurationStore _store;
private IRunnerServer _runnerServer;
private ITerminal _term;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_runnerServer = HostContext.GetService<IRunnerServer>();
Trace.Verbose("Creating _store");
_store = hostContext.GetService<IConfigurationStore>();
Trace.Verbose("store created");
_term = hostContext.GetService<ITerminal>();
}
public bool IsConfigured()
{
bool result = _store.IsConfigured();
Trace.Info($"Is configured: {result}");
return result;
}
public RunnerSettings LoadSettings()
{
Trace.Info(nameof(LoadSettings));
if (!IsConfigured())
{
throw new InvalidOperationException("Not configured");
}
RunnerSettings settings = _store.GetSettings();
Trace.Info("Settings Loaded");
return settings;
}
public async Task ConfigureAsync(CommandSettings command)
{
_term.WriteLine();
_term.WriteLine("--------------------------------------------------------------------------------", ConsoleColor.White);
_term.WriteLine("| ____ _ _ _ _ _ _ _ _ |", ConsoleColor.White);
_term.WriteLine("| / ___(_) |_| | | |_ _| |__ / \\ ___| |_(_) ___ _ __ ___ |", ConsoleColor.White);
_term.WriteLine("| | | _| | __| |_| | | | | '_ \\ / _ \\ / __| __| |/ _ \\| '_ \\/ __| |", ConsoleColor.White);
_term.WriteLine("| | |_| | | |_| _ | |_| | |_) | / ___ \\ (__| |_| | (_) | | | \\__ \\ |", ConsoleColor.White);
_term.WriteLine("| \\____|_|\\__|_| |_|\\__,_|_.__/ /_/ \\_\\___|\\__|_|\\___/|_| |_|___/ |", ConsoleColor.White);
_term.WriteLine("| |", ConsoleColor.White);
_term.Write("| ", ConsoleColor.White);
_term.Write("Self-hosted runner registration", ConsoleColor.Cyan);
_term.WriteLine(" |", ConsoleColor.White);
_term.WriteLine("| |", ConsoleColor.White);
_term.WriteLine("--------------------------------------------------------------------------------", ConsoleColor.White);
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
Trace.Info(nameof(ConfigureAsync));
if (IsConfigured())
{
throw new InvalidOperationException("Cannot configure the runner because it is already configured. To reconfigure the runner, run 'config.cmd remove' or './config.sh remove' first.");
}
// Populate proxy setting from commandline args
var runnerProxy = HostContext.GetService<IRunnerWebProxy>();
bool saveProxySetting = false;
string proxyUrl = command.GetProxyUrl();
if (!string.IsNullOrEmpty(proxyUrl))
{
if (!Uri.IsWellFormedUriString(proxyUrl, UriKind.Absolute))
{
throw new ArgumentOutOfRangeException(nameof(proxyUrl));
}
Trace.Info("Reset proxy base on commandline args.");
string proxyUserName = command.GetProxyUserName();
string proxyPassword = command.GetProxyPassword();
(runnerProxy as RunnerWebProxy).SetupProxy(proxyUrl, proxyUserName, proxyPassword);
saveProxySetting = true;
}
// Populate cert setting from commandline args
var runnerCertManager = HostContext.GetService<IRunnerCertificateManager>();
bool saveCertSetting = false;
bool skipCertValidation = command.GetSkipCertificateValidation();
string caCert = command.GetCACertificate();
string clientCert = command.GetClientCertificate();
string clientCertKey = command.GetClientCertificatePrivateKey();
string clientCertArchive = command.GetClientCertificateArchrive();
string clientCertPassword = command.GetClientCertificatePassword();
// We require all Certificate files are under agent root.
// So we can set ACL correctly when configure as service
if (!string.IsNullOrEmpty(caCert))
{
caCert = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), caCert);
ArgUtil.File(caCert, nameof(caCert));
}
if (!string.IsNullOrEmpty(clientCert) &&
!string.IsNullOrEmpty(clientCertKey) &&
!string.IsNullOrEmpty(clientCertArchive))
{
// Ensure all client cert pieces are there.
clientCert = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), clientCert);
clientCertKey = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), clientCertKey);
clientCertArchive = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), clientCertArchive);
ArgUtil.File(clientCert, nameof(clientCert));
ArgUtil.File(clientCertKey, nameof(clientCertKey));
ArgUtil.File(clientCertArchive, nameof(clientCertArchive));
}
else if (!string.IsNullOrEmpty(clientCert) ||
!string.IsNullOrEmpty(clientCertKey) ||
!string.IsNullOrEmpty(clientCertArchive))
{
// Print out which args are missing.
ArgUtil.NotNullOrEmpty(Constants.Runner.CommandLine.Args.SslClientCert, Constants.Runner.CommandLine.Args.SslClientCert);
ArgUtil.NotNullOrEmpty(Constants.Runner.CommandLine.Args.SslClientCertKey, Constants.Runner.CommandLine.Args.SslClientCertKey);
ArgUtil.NotNullOrEmpty(Constants.Runner.CommandLine.Args.SslClientCertArchive, Constants.Runner.CommandLine.Args.SslClientCertArchive);
}
if (skipCertValidation || !string.IsNullOrEmpty(caCert) || !string.IsNullOrEmpty(clientCert))
{
Trace.Info("Reset runner cert setting base on commandline args.");
(runnerCertManager as RunnerCertificateManager).SetupCertificate(skipCertValidation, caCert, clientCert, clientCertKey, clientCertArchive, clientCertPassword);
saveCertSetting = true;
}
RunnerSettings runnerSettings = new RunnerSettings();
bool isHostedServer = false;
// Loop getting url and creds until you can connect
ICredentialProvider credProvider = null;
VssCredentials creds = null;
_term.WriteSection("Authentication");
while (true)
{
// Get the URL
var inputUrl = command.GetUrl();
if (!inputUrl.Contains("github.com", StringComparison.OrdinalIgnoreCase))
{
runnerSettings.ServerUrl = inputUrl;
// Get the credentials
credProvider = GetCredentialProvider(command, runnerSettings.ServerUrl);
creds = credProvider.GetVssCredentials(HostContext);
Trace.Info("legacy vss cred retrieved");
}
else
{
runnerSettings.GitHubUrl = inputUrl;
var githubToken = command.GetRunnerRegisterToken();
GitHubAuthResult authResult = await GetTenantCredential(inputUrl, githubToken);
runnerSettings.ServerUrl = authResult.TenantUrl;
creds = authResult.ToVssCredentials();
Trace.Info("cred retrieved via GitHub auth");
}
try
{
// Determine the service deployment type based on connection data. (Hosted/OnPremises)
isHostedServer = await IsHostedServer(runnerSettings.ServerUrl, creds);
// Validate can connect.
await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), creds);
_term.WriteLine();
_term.WriteSuccessMessage("Connected to GitHub");
Trace.Info("Test Connection complete.");
break;
}
catch (Exception e) when (!command.Unattended)
{
_term.WriteError(e);
_term.WriteError("Failed to connect. Try again or ctrl-c to quit");
_term.WriteLine();
}
}
// We want to use the native CSP of the platform for storage, so we use the RSACSP directly
RSAParameters publicKey;
var keyManager = HostContext.GetService<IRSAKeyManager>();
using (var rsa = keyManager.CreateKey())
{
publicKey = rsa.ExportParameters(false);
}
_term.WriteSection("Runner Registration");
//Get all the agent pools, and select the first private pool
List<TaskAgentPool> agentPools = await _runnerServer.GetAgentPoolsAsync();
TaskAgentPool agentPool = agentPools?.Where(x => x.IsHosted == false).FirstOrDefault();
if (agentPool == null)
{
throw new TaskAgentPoolNotFoundException($"Could not find any private pool. Contact support.");
}
else
{
Trace.Info("Found a private pool with id {1} and name {2}", agentPool.Id, agentPool.Name);
runnerSettings.PoolId = agentPool.Id;
runnerSettings.PoolName = agentPool.Name;
}
TaskAgent agent;
while (true)
{
runnerSettings.AgentName = command.GetAgentName();
// Get the system capabilities.
Dictionary<string, string> systemCapabilities = await HostContext.GetService<ICapabilitiesManager>().GetCapabilitiesAsync(runnerSettings, CancellationToken.None);
_term.WriteLine();
var agents = await _runnerServer.GetAgentsAsync(runnerSettings.PoolId, runnerSettings.AgentName);
Trace.Verbose("Returns {0} agents", agents.Count);
agent = agents.FirstOrDefault();
if (agent != null)
{
_term.WriteLine("A runner exists with the same name", ConsoleColor.Yellow);
if (command.GetReplace())
{
// Update existing agent with new PublicKey, agent version and SystemCapabilities.
agent = UpdateExistingAgent(agent, publicKey, systemCapabilities);
try
{
agent = await _runnerServer.UpdateAgentAsync(runnerSettings.PoolId, agent);
_term.WriteSuccessMessage("Successfully replaced the runner");
break;
}
catch (Exception e) when (!command.Unattended)
{
_term.WriteError(e);
_term.WriteError("Failed to replace the runner. Try again or ctrl-c to quit");
}
}
else if (command.Unattended)
{
// if not replace and it is unattended config.
throw new TaskAgentExistsException($"Pool {runnerSettings.PoolId} already contains a runner with name {runnerSettings.AgentName}.");
}
}
else
{
// Create a new agent.
agent = CreateNewAgent(runnerSettings.AgentName, publicKey, systemCapabilities);
try
{
agent = await _runnerServer.AddAgentAsync(runnerSettings.PoolId, agent);
_term.WriteSuccessMessage("Runner successfully added");
break;
}
catch (Exception e) when (!command.Unattended)
{
_term.WriteError(e);
_term.WriteError("Failed to add the runner. Try again or ctrl-c to quit");
}
}
}
// Add Agent Id to settings
runnerSettings.AgentId = agent.Id;
// respect the serverUrl resolve by server.
// in case of agent configured using collection url instead of account url.
string agentServerUrl;
if (agent.Properties.TryGetValidatedValue<string>("ServerUrl", out agentServerUrl) &&
!string.IsNullOrEmpty(agentServerUrl))
{
Trace.Info($"Agent server url resolve by server: '{agentServerUrl}'.");
// we need make sure the Schema/Host/Port component of the url remain the same.
UriBuilder inputServerUrl = new UriBuilder(runnerSettings.ServerUrl);
UriBuilder serverReturnedServerUrl = new UriBuilder(agentServerUrl);
if (Uri.Compare(inputServerUrl.Uri, serverReturnedServerUrl.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
{
inputServerUrl.Path = serverReturnedServerUrl.Path;
Trace.Info($"Replace server returned url's scheme://host:port component with user input server url's scheme://host:port: '{inputServerUrl.Uri.AbsoluteUri}'.");
runnerSettings.ServerUrl = inputServerUrl.Uri.AbsoluteUri;
}
else
{
runnerSettings.ServerUrl = agentServerUrl;
}
}
// See if the server supports our OAuth key exchange for credentials
if (agent.Authorization != null &&
agent.Authorization.ClientId != Guid.Empty &&
agent.Authorization.AuthorizationUrl != null)
{
UriBuilder configServerUrl = new UriBuilder(runnerSettings.ServerUrl);
UriBuilder oauthEndpointUrlBuilder = new UriBuilder(agent.Authorization.AuthorizationUrl);
if (!isHostedServer && Uri.Compare(configServerUrl.Uri, oauthEndpointUrlBuilder.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
{
oauthEndpointUrlBuilder.Scheme = configServerUrl.Scheme;
oauthEndpointUrlBuilder.Host = configServerUrl.Host;
oauthEndpointUrlBuilder.Port = configServerUrl.Port;
Trace.Info($"Set oauth endpoint url's scheme://host:port component to match runner configure url's scheme://host:port: '{oauthEndpointUrlBuilder.Uri.AbsoluteUri}'.");
}
var credentialData = new CredentialData
{
Scheme = Constants.Configuration.OAuth,
Data =
{
{ "clientId", agent.Authorization.ClientId.ToString("D") },
{ "authorizationUrl", agent.Authorization.AuthorizationUrl.AbsoluteUri },
{ "oauthEndpointUrl", oauthEndpointUrlBuilder.Uri.AbsoluteUri },
},
};
// Save the negotiated OAuth credential data
_store.SaveCredential(credentialData);
}
else
{
throw new NotSupportedException("Message queue listen OAuth token.");
}
// Testing agent connection, detect any protential connection issue, like local clock skew that cause OAuth token expired.
var credMgr = HostContext.GetService<ICredentialManager>();
VssCredentials credential = credMgr.LoadCredentials();
try
{
await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), credential);
_term.WriteSuccessMessage("Runner connection is good");
}
catch (VssOAuthTokenRequestException ex) when (ex.Message.Contains("Current server time is"))
{
// there are two exception messages server send that indicate clock skew.
// 1. The bearer token expired on {jwt.ValidTo}. Current server time is {DateTime.UtcNow}.
// 2. The bearer token is not valid until {jwt.ValidFrom}. Current server time is {DateTime.UtcNow}.
Trace.Error("Catch exception during test agent connection.");
Trace.Error(ex);
throw new Exception("The local machine's clock may be out of sync with the server time by more than five minutes. Please sync your clock with your domain or internet time and try again.");
}
_term.WriteSection("Runner settings");
// We will Combine() what's stored with root. Defaults to string a relative path
runnerSettings.WorkFolder = command.GetWork();
// notificationPipeName for Hosted agent provisioner.
runnerSettings.NotificationPipeName = command.GetNotificationPipeName();
runnerSettings.MonitorSocketAddress = command.GetMonitorSocketAddress();
runnerSettings.NotificationSocketAddress = command.GetNotificationSocketAddress();
_store.SaveSettings(runnerSettings);
if (saveProxySetting)
{
Trace.Info("Save proxy setting to disk.");
(runnerProxy as RunnerWebProxy).SaveProxySetting();
}
if (saveCertSetting)
{
Trace.Info("Save agent cert setting to disk.");
(runnerCertManager as RunnerCertificateManager).SaveCertificateSetting();
}
_term.WriteLine();
_term.WriteSuccessMessage("Settings Saved.");
_term.WriteLine();
bool saveRuntimeOptions = false;
var runtimeOptions = new RunnerRuntimeOptions();
#if OS_WINDOWS
if (command.GitUseSChannel)
{
saveRuntimeOptions = true;
runtimeOptions.GitUseSecureChannel = true;
}
#endif
if (saveRuntimeOptions)
{
Trace.Info("Save agent runtime options to disk.");
_store.SaveRunnerRuntimeOptions(runtimeOptions);
}
#if OS_WINDOWS
// config windows service
bool runAsService = command.GetRunAsService();
if (runAsService)
{
Trace.Info("Configuring to run the agent as service");
var serviceControlManager = HostContext.GetService<IWindowsServiceControlManager>();
serviceControlManager.ConfigureService(runnerSettings, command);
}
#elif OS_LINUX || OS_OSX
// generate service config script for OSX and Linux, GenerateScripts() will no-opt on windows.
var serviceControlManager = HostContext.GetService<ILinuxServiceControlManager>();
serviceControlManager.GenerateScripts(runnerSettings);
#endif
}
public async Task UnconfigureAsync(CommandSettings command)
{
ArgUtil.Equal(RunMode.Normal, HostContext.RunMode, nameof(HostContext.RunMode));
string currentAction = string.Empty;
_term.WriteSection("Runner removal");
try
{
//stop, uninstall service and remove service config file
if (_store.IsServiceConfigured())
{
currentAction = "Removing service";
_term.WriteLine(currentAction);
#if OS_WINDOWS
var serviceControlManager = HostContext.GetService<IWindowsServiceControlManager>();
serviceControlManager.UnconfigureService();
_term.WriteLine();
_term.WriteSuccessMessage("Runner service removed");
#elif OS_LINUX
// unconfig system D service first
throw new Exception("Unconfigure service first");
#elif OS_OSX
// unconfig osx service first
throw new Exception("Unconfigure service first");
#endif
}
//delete agent from the server
currentAction = "Removing runner from the server";
bool isConfigured = _store.IsConfigured();
bool hasCredentials = _store.HasCredentials();
if (isConfigured && hasCredentials)
{
RunnerSettings settings = _store.GetSettings();
var credentialManager = HostContext.GetService<ICredentialManager>();
// Get the credentials
VssCredentials creds = null;
if (string.IsNullOrEmpty(settings.GitHubUrl))
{
var credProvider = GetCredentialProvider(command, settings.ServerUrl);
creds = credProvider.GetVssCredentials(HostContext);
Trace.Info("legacy vss cred retrieved");
}
else
{
var githubToken = command.GetToken();
GitHubAuthResult authResult = await GetTenantCredential(settings.GitHubUrl, githubToken);
creds = authResult.ToVssCredentials();
Trace.Info("cred retrieved via GitHub auth");
}
// Determine the service deployment type based on connection data. (Hosted/OnPremises)
bool isHostedServer = await IsHostedServer(settings.ServerUrl, creds);
await _runnerServer.ConnectAsync(new Uri(settings.ServerUrl), creds);
var agents = await _runnerServer.GetAgentsAsync(settings.PoolId, settings.AgentName);
Trace.Verbose("Returns {0} agents", agents.Count);
TaskAgent agent = agents.FirstOrDefault();
if (agent == null)
{
_term.WriteLine("Does not exist. Skipping " + currentAction);
}
else
{
await _runnerServer.DeleteAgentAsync(settings.PoolId, settings.AgentId);
_term.WriteLine();
_term.WriteSuccessMessage("Runner removed successfully");
}
}
else
{
_term.WriteLine("Cannot connect to server, because config files are missing. Skipping removing runner from the server.");
}
//delete credential config files
currentAction = "Removing .credentials";
if (hasCredentials)
{
_store.DeleteCredential();
var keyManager = HostContext.GetService<IRSAKeyManager>();
keyManager.DeleteKey();
_term.WriteSuccessMessage("Removed .credentials");
}
else
{
_term.WriteLine("Does not exist. Skipping " + currentAction);
}
//delete settings config file
currentAction = "Removing .runner";
if (isConfigured)
{
// delete proxy setting
(HostContext.GetService<IRunnerWebProxy>() as RunnerWebProxy).DeleteProxySetting();
// delete agent cert setting
(HostContext.GetService<IRunnerCertificateManager>() as RunnerCertificateManager).DeleteCertificateSetting();
// delete agent runtime option
_store.DeleteRunnerRuntimeOptions();
_store.DeleteSettings();
_term.WriteSuccessMessage("Removed .runner");
}
else
{
_term.WriteLine("Does not exist. Skipping " + currentAction);
}
}
catch (Exception)
{
_term.WriteError("Failed: " + currentAction);
throw;
}
_term.WriteLine();
}
private ICredentialProvider GetCredentialProvider(CommandSettings command, string serverUrl)
{
Trace.Info(nameof(GetCredentialProvider));
var credentialManager = HostContext.GetService<ICredentialManager>();
string authType = command.GetAuth(defaultValue: Constants.Configuration.AAD);
// Create the credential.
Trace.Info("Creating credential for auth: {0}", authType);
var provider = credentialManager.GetCredentialProvider(authType);
if (provider.RequireInteractive && command.Unattended)
{
throw new NotSupportedException($"Authentication type '{authType}' is not supported for unattended configuration.");
}
provider.EnsureCredential(HostContext, command, serverUrl);
return provider;
}
private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, Dictionary<string, string> systemCapabilities)
{
ArgUtil.NotNull(agent, nameof(agent));
agent.Authorization = new TaskAgentAuthorization
{
PublicKey = new TaskAgentPublicKey(publicKey.Exponent, publicKey.Modulus),
};
// update - update instead of delete so we don't lose user capabilities etc...
agent.Version = BuildConstants.RunnerPackage.Version;
agent.OSDescription = RuntimeInformation.OSDescription;
foreach (KeyValuePair<string, string> capability in systemCapabilities)
{
agent.SystemCapabilities[capability.Key] = capability.Value ?? string.Empty;
}
return agent;
}
private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey, Dictionary<string, string> systemCapabilities)
{
TaskAgent agent = new TaskAgent(agentName)
{
Authorization = new TaskAgentAuthorization
{
PublicKey = new TaskAgentPublicKey(publicKey.Exponent, publicKey.Modulus),
},
MaxParallelism = 1,
Version = BuildConstants.RunnerPackage.Version,
OSDescription = RuntimeInformation.OSDescription,
};
foreach (KeyValuePair<string, string> capability in systemCapabilities)
{
agent.SystemCapabilities[capability.Key] = capability.Value ?? string.Empty;
}
return agent;
}
private async Task<bool> IsHostedServer(string serverUrl, VssCredentials credentials)
{
// Determine the service deployment type based on connection data. (Hosted/OnPremises)
var locationServer = HostContext.GetService<ILocationServer>();
VssConnection connection = VssUtil.CreateConnection(new Uri(serverUrl), credentials);
await locationServer.ConnectAsync(connection);
try
{
var connectionData = await locationServer.GetConnectionDataAsync();
Trace.Info($"Server deployment type: {connectionData.DeploymentType}");
return connectionData.DeploymentType.HasFlag(DeploymentFlags.Hosted);
}
catch (Exception ex)
{
// Since the DeploymentType is Enum, deserialization exception means there is a new Enum member been added.
// It's more likely to be Hosted since OnPremises is always behind and customer can update their agent if are on-prem
Trace.Error(ex);
return true;
}
}
private async Task<GitHubAuthResult> GetTenantCredential(string githubUrl, string githubToken)
{
var gitHubUrl = new UriBuilder(githubUrl);
var githubApiUrl = $"https://api.github.com/repos/{gitHubUrl.Path.Trim('/')}/actions-runners/registration";
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("RemoteAuth", githubToken);
httpClient.DefaultRequestHeaders.UserAgent.Add(HostContext.UserAgent);
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github.shuri-preview+json"));
var response = await httpClient.PostAsync(githubApiUrl, new StringContent("", null, "application/json"));
if (response.IsSuccessStatusCode)
{
Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var jsonResponse = await response.Content.ReadAsStringAsync();
return StringUtil.ConvertFromJson<GitHubAuthResult>(jsonResponse);
}
else
{
_term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var errorResponse = await response.Content.ReadAsStringAsync();
_term.WriteError(errorResponse);
response.EnsureSuccessStatusCode();
return null;
}
}
}
}
}

View File

@@ -0,0 +1,91 @@
using System;
using System.Collections.Generic;
using System.Runtime.Serialization;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
using GitHub.Services.Common;
using GitHub.Services.OAuth;
namespace GitHub.Runner.Listener.Configuration
{
// TODO: Refactor extension manager to enable using it from the agent process.
[ServiceLocator(Default = typeof(CredentialManager))]
public interface ICredentialManager : IRunnerService
{
ICredentialProvider GetCredentialProvider(string credType);
VssCredentials LoadCredentials();
}
public class CredentialManager : RunnerService, ICredentialManager
{
public static readonly Dictionary<string, Type> CredentialTypes = new Dictionary<string, Type>(StringComparer.OrdinalIgnoreCase)
{
{ Constants.Configuration.AAD, typeof(AadDeviceCodeAccessToken)},
{ Constants.Configuration.PAT, typeof(PersonalAccessToken)},
{ Constants.Configuration.OAuth, typeof(OAuthCredential)},
{ Constants.Configuration.OAuthAccessToken, typeof(OAuthAccessTokenCredential)},
};
public ICredentialProvider GetCredentialProvider(string credType)
{
Trace.Info(nameof(GetCredentialProvider));
Trace.Info("Creating type {0}", credType);
if (!CredentialTypes.ContainsKey(credType))
{
throw new ArgumentException("Invalid Credential Type");
}
Trace.Info("Creating credential type: {0}", credType);
var creds = Activator.CreateInstance(CredentialTypes[credType]) as ICredentialProvider;
Trace.Verbose("Created credential type");
return creds;
}
public VssCredentials LoadCredentials()
{
IConfigurationStore store = HostContext.GetService<IConfigurationStore>();
if (!store.HasCredentials())
{
throw new InvalidOperationException("Credentials not stored. Must reconfigure.");
}
CredentialData credData = store.GetCredentials();
ICredentialProvider credProv = GetCredentialProvider(credData.Scheme);
credProv.CredentialData = credData;
VssCredentials creds = credProv.GetVssCredentials(HostContext);
return creds;
}
}
[DataContract]
public sealed class GitHubAuthResult
{
[DataMember(Name = "url")]
public string TenantUrl { get; set; }
[DataMember(Name = "token_schema")]
public string TokenSchema { get; set; }
[DataMember(Name = "token")]
public string Token { get; set; }
public VssCredentials ToVssCredentials()
{
ArgUtil.NotNullOrEmpty(TokenSchema, nameof(TokenSchema));
ArgUtil.NotNullOrEmpty(Token, nameof(Token));
if (string.Equals(TokenSchema, "OAuthAccessToken", StringComparison.OrdinalIgnoreCase))
{
return new VssCredentials(null, new VssOAuthAccessTokenCredential(Token), CredentialPromptType.DoNotPrompt);
}
else
{
throw new NotSupportedException($"Not supported token schema: {TokenSchema}");
}
}
}
}

View File

@@ -0,0 +1,231 @@
using System;
using System.Diagnostics;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using Microsoft.IdentityModel.Clients.ActiveDirectory;
using GitHub.Runner.Common.Util;
using GitHub.Services.Client;
using GitHub.Services.Common;
using GitHub.Services.WebApi;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
using GitHub.Services.OAuth;
namespace GitHub.Runner.Listener.Configuration
{
public interface ICredentialProvider
{
Boolean RequireInteractive { get; }
CredentialData CredentialData { get; set; }
VssCredentials GetVssCredentials(IHostContext context);
void EnsureCredential(IHostContext context, CommandSettings command, string serverUrl);
}
public abstract class CredentialProvider : ICredentialProvider
{
public CredentialProvider(string scheme)
{
CredentialData = new CredentialData();
CredentialData.Scheme = scheme;
}
public virtual Boolean RequireInteractive => false;
public CredentialData CredentialData { get; set; }
public abstract VssCredentials GetVssCredentials(IHostContext context);
public abstract void EnsureCredential(IHostContext context, CommandSettings command, string serverUrl);
}
public sealed class AadDeviceCodeAccessToken : CredentialProvider
{
private string _azureDevOpsClientId = "97877f11-0fc6-4aee-b1ff-febb0519dd00";
public override Boolean RequireInteractive => true;
public AadDeviceCodeAccessToken() : base(Constants.Configuration.AAD) { }
public override VssCredentials GetVssCredentials(IHostContext context)
{
ArgUtil.NotNull(context, nameof(context));
Tracing trace = context.GetTrace(nameof(AadDeviceCodeAccessToken));
trace.Info(nameof(GetVssCredentials));
ArgUtil.NotNull(CredentialData, nameof(CredentialData));
CredentialData.Data.TryGetValue(Constants.Runner.CommandLine.Args.Url, out string serverUrl);
ArgUtil.NotNullOrEmpty(serverUrl, nameof(serverUrl));
var tenantAuthorityUrl = GetTenantAuthorityUrl(context, serverUrl);
if (tenantAuthorityUrl == null)
{
throw new NotSupportedException($"'{serverUrl}' is not backed by Azure Active Directory.");
}
LoggerCallbackHandler.LogCallback = ((LogLevel level, string message, bool containsPii) =>
{
switch (level)
{
case LogLevel.Information:
trace.Info(message);
break;
case LogLevel.Error:
trace.Error(message);
break;
case LogLevel.Warning:
trace.Warning(message);
break;
default:
trace.Verbose(message);
break;
}
});
LoggerCallbackHandler.UseDefaultLogging = false;
AuthenticationContext ctx = new AuthenticationContext(tenantAuthorityUrl.AbsoluteUri);
var queryParameters = $"redirect_uri={Uri.EscapeDataString(new Uri(serverUrl).GetLeftPart(UriPartial.Authority))}";
DeviceCodeResult codeResult = ctx.AcquireDeviceCodeAsync("https://management.core.windows.net/", _azureDevOpsClientId, queryParameters).GetAwaiter().GetResult();
var term = context.GetService<ITerminal>();
term.WriteLine($"Please finish AAD device code flow in browser ({codeResult.VerificationUrl}), user code: {codeResult.UserCode}");
if (string.Equals(CredentialData.Data[Constants.Runner.CommandLine.Flags.LaunchBrowser], bool.TrueString, StringComparison.OrdinalIgnoreCase))
{
try
{
#if OS_WINDOWS
Process.Start(new ProcessStartInfo() { FileName = codeResult.VerificationUrl, UseShellExecute = true });
#elif OS_LINUX
Process.Start(new ProcessStartInfo() { FileName = "xdg-open", Arguments = codeResult.VerificationUrl });
#else
Process.Start(new ProcessStartInfo() { FileName = "open", Arguments = codeResult.VerificationUrl });
#endif
}
catch (Exception ex)
{
// not able to open browser, ex: xdg-open/open is not installed.
trace.Error(ex);
term.WriteLine($"Fail to open browser. {codeResult.Message}");
}
}
AuthenticationResult authResult = ctx.AcquireTokenByDeviceCodeAsync(codeResult).GetAwaiter().GetResult();
ArgUtil.NotNull(authResult, nameof(authResult));
trace.Info($"receive AAD auth result with {authResult.AccessTokenType} token");
var aadCred = new VssAadCredential(new VssAadToken(authResult));
VssCredentials creds = new VssCredentials(null, aadCred, CredentialPromptType.DoNotPrompt);
trace.Info("cred created");
return creds;
}
public override void EnsureCredential(IHostContext context, CommandSettings command, string serverUrl)
{
ArgUtil.NotNull(context, nameof(context));
Tracing trace = context.GetTrace(nameof(AadDeviceCodeAccessToken));
trace.Info(nameof(EnsureCredential));
ArgUtil.NotNull(command, nameof(command));
CredentialData.Data[Constants.Runner.CommandLine.Args.Url] = serverUrl;
CredentialData.Data[Constants.Runner.CommandLine.Flags.LaunchBrowser] = command.GetAutoLaunchBrowser().ToString();
}
private Uri GetTenantAuthorityUrl(IHostContext context, string serverUrl)
{
using (var client = new HttpClient(context.CreateHttpClientHandler()))
{
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
client.DefaultRequestHeaders.Add("X-TFS-FedAuthRedirect", "Suppress");
client.DefaultRequestHeaders.UserAgent.Clear();
client.DefaultRequestHeaders.UserAgent.AddRange(VssClientHttpRequestSettings.Default.UserAgent);
var requestMessage = new HttpRequestMessage(HttpMethod.Head, $"{serverUrl.Trim('/')}/_apis/connectiondata");
var response = client.SendAsync(requestMessage).GetAwaiter().GetResult();
// Get the tenant from the Login URL, MSA backed accounts will not return `Bearer` www-authenticate header.
var bearerResult = response.Headers.WwwAuthenticate.Where(p => p.Scheme.Equals("Bearer", StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
if (bearerResult != null && bearerResult.Parameter.StartsWith("authorization_uri=", StringComparison.OrdinalIgnoreCase))
{
var authorizationUri = bearerResult.Parameter.Substring("authorization_uri=".Length);
if (Uri.TryCreate(authorizationUri, UriKind.Absolute, out Uri aadTenantUrl))
{
return aadTenantUrl;
}
}
return null;
}
}
}
public sealed class OAuthAccessTokenCredential : CredentialProvider
{
public OAuthAccessTokenCredential() : base(Constants.Configuration.OAuthAccessToken) { }
public override VssCredentials GetVssCredentials(IHostContext context)
{
ArgUtil.NotNull(context, nameof(context));
Tracing trace = context.GetTrace(nameof(OAuthAccessTokenCredential));
trace.Info(nameof(GetVssCredentials));
ArgUtil.NotNull(CredentialData, nameof(CredentialData));
string token;
if (!CredentialData.Data.TryGetValue(Constants.Runner.CommandLine.Args.Token, out token))
{
token = null;
}
ArgUtil.NotNullOrEmpty(token, nameof(token));
trace.Info("token retrieved: {0} chars", token.Length);
VssCredentials creds = new VssCredentials(null, new VssOAuthAccessTokenCredential(token), CredentialPromptType.DoNotPrompt);
trace.Info("cred created");
return creds;
}
public override void EnsureCredential(IHostContext context, CommandSettings command, string serverUrl)
{
ArgUtil.NotNull(context, nameof(context));
Tracing trace = context.GetTrace(nameof(OAuthAccessTokenCredential));
trace.Info(nameof(EnsureCredential));
ArgUtil.NotNull(command, nameof(command));
CredentialData.Data[Constants.Runner.CommandLine.Args.Token] = command.GetToken();
}
}
public sealed class PersonalAccessToken : CredentialProvider
{
public PersonalAccessToken() : base(Constants.Configuration.PAT) { }
public override VssCredentials GetVssCredentials(IHostContext context)
{
ArgUtil.NotNull(context, nameof(context));
Tracing trace = context.GetTrace(nameof(PersonalAccessToken));
trace.Info(nameof(GetVssCredentials));
ArgUtil.NotNull(CredentialData, nameof(CredentialData));
string token;
if (!CredentialData.Data.TryGetValue(Constants.Runner.CommandLine.Args.Token, out token))
{
token = null;
}
ArgUtil.NotNullOrEmpty(token, nameof(token));
trace.Info("token retrieved: {0} chars", token.Length);
// PAT uses a basic credential
VssBasicCredential basicCred = new VssBasicCredential("ActionsRunner", token);
VssCredentials creds = new VssCredentials(null, basicCred, CredentialPromptType.DoNotPrompt);
trace.Info("cred created");
return creds;
}
public override void EnsureCredential(IHostContext context, CommandSettings command, string serverUrl)
{
ArgUtil.NotNull(context, nameof(context));
Tracing trace = context.GetTrace(nameof(PersonalAccessToken));
trace.Info(nameof(EnsureCredential));
ArgUtil.NotNull(command, nameof(command));
CredentialData.Data[Constants.Runner.CommandLine.Args.Token] = command.GetToken();
}
}
}

View File

@@ -0,0 +1,108 @@
using System;
using System.Runtime.Serialization;
using System.Security.Cryptography;
using GitHub.Runner.Common;
namespace GitHub.Runner.Listener.Configuration
{
/// <summary>
/// Manages an RSA key for the agent using the most appropriate store for the target platform.
/// </summary>
#if OS_WINDOWS
[ServiceLocator(Default = typeof(RSAEncryptedFileKeyManager))]
#else
[ServiceLocator(Default = typeof(RSAFileKeyManager))]
#endif
public interface IRSAKeyManager : IRunnerService
{
/// <summary>
/// Creates a new <c>RSACryptoServiceProvider</c> instance for the current agent. If a key file is found then the current
/// key is returned to the caller.
/// </summary>
/// <returns>An <c>RSACryptoServiceProvider</c> instance representing the key for the agent</returns>
RSACryptoServiceProvider CreateKey();
/// <summary>
/// Deletes the RSA key managed by the key manager.
/// </summary>
void DeleteKey();
/// <summary>
/// Gets the <c>RSACryptoServiceProvider</c> instance currently stored by the key manager.
/// </summary>
/// <returns>An <c>RSACryptoServiceProvider</c> instance representing the key for the agent</returns>
/// <exception cref="CryptographicException">No key exists in the store</exception>
RSACryptoServiceProvider GetKey();
}
// Newtonsoft 10 is not working properly with dotnet RSAParameters class
// RSAParameters has fields marked as [NonSerialized] which cause we loss those fields after serialize to JSON
// https://github.com/JamesNK/Newtonsoft.Json/issues/1517
// https://github.com/dotnet/corefx/issues/23847
// As workaround, we create our own RSAParameters class without any [NonSerialized] attributes.
[Serializable]
internal class RSAParametersSerializable : ISerializable
{
private RSAParameters _rsaParameters;
public RSAParameters RSAParameters
{
get
{
return _rsaParameters;
}
}
public RSAParametersSerializable(RSAParameters rsaParameters)
{
_rsaParameters = rsaParameters;
}
private RSAParametersSerializable()
{
}
public byte[] D { get { return _rsaParameters.D; } set { _rsaParameters.D = value; } }
public byte[] DP { get { return _rsaParameters.DP; } set { _rsaParameters.DP = value; } }
public byte[] DQ { get { return _rsaParameters.DQ; } set { _rsaParameters.DQ = value; } }
public byte[] Exponent { get { return _rsaParameters.Exponent; } set { _rsaParameters.Exponent = value; } }
public byte[] InverseQ { get { return _rsaParameters.InverseQ; } set { _rsaParameters.InverseQ = value; } }
public byte[] Modulus { get { return _rsaParameters.Modulus; } set { _rsaParameters.Modulus = value; } }
public byte[] P { get { return _rsaParameters.P; } set { _rsaParameters.P = value; } }
public byte[] Q { get { return _rsaParameters.Q; } set { _rsaParameters.Q = value; } }
public RSAParametersSerializable(SerializationInfo information, StreamingContext context)
{
_rsaParameters = new RSAParameters()
{
D = (byte[])information.GetValue("d", typeof(byte[])),
DP = (byte[])information.GetValue("dp", typeof(byte[])),
DQ = (byte[])information.GetValue("dq", typeof(byte[])),
Exponent = (byte[])information.GetValue("exponent", typeof(byte[])),
InverseQ = (byte[])information.GetValue("inverseQ", typeof(byte[])),
Modulus = (byte[])information.GetValue("modulus", typeof(byte[])),
P = (byte[])information.GetValue("p", typeof(byte[])),
Q = (byte[])information.GetValue("q", typeof(byte[]))
};
}
public void GetObjectData(SerializationInfo info, StreamingContext context)
{
info.AddValue("d", _rsaParameters.D);
info.AddValue("dp", _rsaParameters.DP);
info.AddValue("dq", _rsaParameters.DQ);
info.AddValue("exponent", _rsaParameters.Exponent);
info.AddValue("inverseQ", _rsaParameters.InverseQ);
info.AddValue("modulus", _rsaParameters.Modulus);
info.AddValue("p", _rsaParameters.P);
info.AddValue("q", _rsaParameters.Q);
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,49 @@
using System;
using GitHub.Runner.Common;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using GitHub.Services.Common;
using GitHub.Services.OAuth;
using GitHub.Services.WebApi;
namespace GitHub.Runner.Listener.Configuration
{
public class OAuthCredential : CredentialProvider
{
public OAuthCredential()
: base(Constants.Configuration.OAuth)
{
}
public override void EnsureCredential(
IHostContext context,
CommandSettings command,
String serverUrl)
{
// Nothing to verify here
}
public override VssCredentials GetVssCredentials(IHostContext context)
{
var clientId = this.CredentialData.Data.GetValueOrDefault("clientId", null);
var authorizationUrl = this.CredentialData.Data.GetValueOrDefault("authorizationUrl", null);
// For back compat with .credential file that doesn't has 'oauthEndpointUrl' section
var oathEndpointUrl = this.CredentialData.Data.GetValueOrDefault("oauthEndpointUrl", authorizationUrl);
ArgUtil.NotNullOrEmpty(clientId, nameof(clientId));
ArgUtil.NotNullOrEmpty(authorizationUrl, nameof(authorizationUrl));
// We expect the key to be in the machine store at this point. Configuration should have set all of
// this up correctly so we can use the key to generate access tokens.
var keyManager = context.GetService<IRSAKeyManager>();
var signingCredentials = VssSigningCredentials.Create(() => keyManager.GetKey());
var clientCredential = new VssOAuthJwtBearerClientCredential(clientId, authorizationUrl, signingCredentials);
var agentCredential = new VssOAuthCredential(new Uri(oathEndpointUrl, UriKind.Absolute), VssOAuthGrant.ClientCredentials, clientCredential);
// Construct a credentials cache with a single OAuth credential for communication. The windows credential
// is explicitly set to null to ensure we never do that negotiation.
return new VssCredentials(null, agentCredential, CredentialPromptType.DoNotPrompt);
}
}
}

View File

@@ -0,0 +1,59 @@
#if OS_OSX
using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Listener.Configuration
{
public class OsxServiceControlManager : ServiceControlManager, ILinuxServiceControlManager
{
// This is the name you would see when you do `systemctl list-units | grep runner`
private const string _svcNamePattern = "actions.runner.{0}.{1}.{2}";
private const string _svcDisplayPattern = "GitHub Actions Runner ({0}.{1}.{2})";
private const string _shTemplate = "darwin.svc.sh.template";
private const string _svcShName = "svc.sh";
public void GenerateScripts(RunnerSettings settings)
{
Trace.Entering();
string serviceName;
string serviceDisplayName;
CalculateServiceName(settings, _svcNamePattern, _svcDisplayPattern, out serviceName, out serviceDisplayName);
try
{
string svcShPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), _svcShName);
// TODO: encoding?
// TODO: Loc strings formatted into MSG_xxx vars in shellscript
string svcShContent = File.ReadAllText(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), _shTemplate));
var tokensToReplace = new Dictionary<string, string>
{
{ "{{SvcDescription}}", serviceDisplayName },
{ "{{SvcNameVar}}", serviceName }
};
svcShContent = tokensToReplace.Aggregate(
svcShContent,
(current, item) => current.Replace(item.Key, item.Value));
//TODO: encoding?
File.WriteAllText(svcShPath, svcShContent);
var unixUtil = HostContext.CreateService<IUnixUtil>();
unixUtil.ChmodAsync("755", svcShPath).GetAwaiter().GetResult();
}
catch (Exception e)
{
Trace.Error(e);
throw;
}
}
}
}
#endif

View File

@@ -0,0 +1,117 @@
using GitHub.Runner.Common;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using System;
namespace GitHub.Runner.Listener.Configuration
{
[ServiceLocator(Default = typeof(PromptManager))]
public interface IPromptManager : IRunnerService
{
bool ReadBool(
string argName,
string description,
bool defaultValue,
bool unattended);
string ReadValue(
string argName,
string description,
bool secret,
string defaultValue,
Func<String, bool> validator,
bool unattended);
}
public sealed class PromptManager : RunnerService, IPromptManager
{
private ITerminal _terminal;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_terminal = HostContext.GetService<ITerminal>();
}
public bool ReadBool(
string argName,
string description,
bool defaultValue,
bool unattended)
{
string answer = ReadValue(
argName: argName,
description: description,
secret: false,
defaultValue: defaultValue ? "Y" : "N",
validator: Validators.BoolValidator,
unattended: unattended);
return String.Equals(answer, "true", StringComparison.OrdinalIgnoreCase) ||
String.Equals(answer, "Y", StringComparison.CurrentCultureIgnoreCase);
}
public string ReadValue(
string argName,
string description,
bool secret,
string defaultValue,
Func<string, bool> validator,
bool unattended)
{
Trace.Info(nameof(ReadValue));
ArgUtil.NotNull(validator, nameof(validator));
string value = string.Empty;
// Check if unattended.
if (unattended)
{
// Return the default value if specified.
if (!string.IsNullOrEmpty(defaultValue))
{
return defaultValue;
}
// Otherwise throw.
throw new Exception($"Invalid configuration provided for {argName}. Terminating unattended configuration.");
}
// Prompt until a valid value is read.
while (true)
{
// Write the message prompt.
_terminal.Write($"{description} ", ConsoleColor.White);
if(!string.IsNullOrEmpty(defaultValue))
{
_terminal.Write($"[press Enter for {defaultValue}] ");
}
// Read and trim the value.
value = secret ? _terminal.ReadSecret() : _terminal.ReadLine();
value = value?.Trim() ?? string.Empty;
// Return the default if not specified.
if (string.IsNullOrEmpty(value) && !string.IsNullOrEmpty(defaultValue))
{
Trace.Info($"Falling back to the default: '{defaultValue}'");
return defaultValue;
}
// Return the value if it is not empty and it is valid.
// Otherwise try the loop again.
if (!string.IsNullOrEmpty(value))
{
if (validator(value))
{
return value;
}
else
{
Trace.Info("Invalid value.");
_terminal.WriteLine("Entered value is invalid", ConsoleColor.Yellow);
}
}
}
}
}
}

View File

@@ -0,0 +1,87 @@
#if OS_WINDOWS
using System.IO;
using System.Security.Cryptography;
using System.Text;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Listener.Configuration
{
public class RSAEncryptedFileKeyManager : RunnerService, IRSAKeyManager
{
private string _keyFile;
private IHostContext _context;
public RSACryptoServiceProvider CreateKey()
{
RSACryptoServiceProvider rsa = null;
if (!File.Exists(_keyFile))
{
Trace.Info("Creating new RSA key using 2048-bit key length");
rsa = new RSACryptoServiceProvider(2048);
// Now write the parameters to disk
SaveParameters(rsa.ExportParameters(true));
Trace.Info("Successfully saved RSA key parameters to file {0}", _keyFile);
}
else
{
Trace.Info("Found existing RSA key parameters file {0}", _keyFile);
rsa = new RSACryptoServiceProvider();
rsa.ImportParameters(LoadParameters());
}
return rsa;
}
public void DeleteKey()
{
if (File.Exists(_keyFile))
{
Trace.Info("Deleting RSA key parameters file {0}", _keyFile);
File.Delete(_keyFile);
}
}
public RSACryptoServiceProvider GetKey()
{
if (!File.Exists(_keyFile))
{
throw new CryptographicException($"RSA key file {_keyFile} was not found");
}
Trace.Info("Loading RSA key parameters from file {0}", _keyFile);
var rsa = new RSACryptoServiceProvider();
rsa.ImportParameters(LoadParameters());
return rsa;
}
private RSAParameters LoadParameters()
{
var encryptedBytes = File.ReadAllBytes(_keyFile);
var parametersString = Encoding.UTF8.GetString(ProtectedData.Unprotect(encryptedBytes, null, DataProtectionScope.LocalMachine));
return StringUtil.ConvertFromJson<RSAParametersSerializable>(parametersString).RSAParameters;
}
private void SaveParameters(RSAParameters parameters)
{
var parametersString = StringUtil.ConvertToJson(new RSAParametersSerializable(parameters));
var encryptedBytes = ProtectedData.Protect(Encoding.UTF8.GetBytes(parametersString), null, DataProtectionScope.LocalMachine);
File.WriteAllBytes(_keyFile, encryptedBytes);
File.SetAttributes(_keyFile, File.GetAttributes(_keyFile) | FileAttributes.Hidden);
}
void IRunnerService.Initialize(IHostContext context)
{
base.Initialize(context);
_context = context;
_keyFile = context.GetConfigFile(WellKnownConfigFile.RSACredentials);
}
}
}
#endif

View File

@@ -0,0 +1,97 @@
#if OS_LINUX || OS_OSX
using System;
using System.IO;
using System.Security.Cryptography;
using System.Threading;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Listener.Configuration
{
public class RSAFileKeyManager : RunnerService, IRSAKeyManager
{
private string _keyFile;
private IHostContext _context;
public RSACryptoServiceProvider CreateKey()
{
RSACryptoServiceProvider rsa = null;
if (!File.Exists(_keyFile))
{
Trace.Info("Creating new RSA key using 2048-bit key length");
rsa = new RSACryptoServiceProvider(2048);
// Now write the parameters to disk
IOUtil.SaveObject(new RSAParametersSerializable(rsa.ExportParameters(true)), _keyFile);
Trace.Info("Successfully saved RSA key parameters to file {0}", _keyFile);
// Try to lock down the credentials_key file to the owner/group
var chmodPath = WhichUtil.Which("chmod", trace: Trace);
if (!String.IsNullOrEmpty(chmodPath))
{
var arguments = $"600 {new FileInfo(_keyFile).FullName}";
using (var invoker = _context.CreateService<IProcessInvoker>())
{
var exitCode = invoker.ExecuteAsync(HostContext.GetDirectory(WellKnownDirectory.Root), chmodPath, arguments, null, default(CancellationToken)).GetAwaiter().GetResult();
if (exitCode == 0)
{
Trace.Info("Successfully set permissions for RSA key parameters file {0}", _keyFile);
}
else
{
Trace.Warning("Unable to succesfully set permissions for RSA key parameters file {0}. Received exit code {1} from {2}", _keyFile, exitCode, chmodPath);
}
}
}
else
{
Trace.Warning("Unable to locate chmod to set permissions for RSA key parameters file {0}.", _keyFile);
}
}
else
{
Trace.Info("Found existing RSA key parameters file {0}", _keyFile);
rsa = new RSACryptoServiceProvider();
rsa.ImportParameters(IOUtil.LoadObject<RSAParametersSerializable>(_keyFile).RSAParameters);
}
return rsa;
}
public void DeleteKey()
{
if (File.Exists(_keyFile))
{
Trace.Info("Deleting RSA key parameters file {0}", _keyFile);
File.Delete(_keyFile);
}
}
public RSACryptoServiceProvider GetKey()
{
if (!File.Exists(_keyFile))
{
throw new CryptographicException($"RSA key file {_keyFile} was not found");
}
Trace.Info("Loading RSA key parameters from file {0}", _keyFile);
var parameters = IOUtil.LoadObject<RSAParametersSerializable>(_keyFile).RSAParameters;
var rsa = new RSACryptoServiceProvider();
rsa.ImportParameters(parameters);
return rsa;
}
void IRunnerService.Initialize(IHostContext context)
{
base.Initialize(context);
_context = context;
_keyFile = context.GetConfigFile(WellKnownConfigFile.RSACredentials);
}
}
}
#endif

View File

@@ -0,0 +1,63 @@
using System;
using System.Linq;
using GitHub.Runner.Common;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Listener.Configuration
{
#if OS_WINDOWS
[ServiceLocator(Default = typeof(WindowsServiceControlManager))]
public interface IWindowsServiceControlManager : IRunnerService
{
void ConfigureService(RunnerSettings settings, CommandSettings command);
void UnconfigureService();
}
#endif
#if !OS_WINDOWS
#if OS_LINUX
[ServiceLocator(Default = typeof(SystemDControlManager))]
#elif OS_OSX
[ServiceLocator(Default = typeof(OsxServiceControlManager))]
#endif
public interface ILinuxServiceControlManager : IRunnerService
{
void GenerateScripts(RunnerSettings settings);
}
#endif
public class ServiceControlManager : RunnerService
{
public void CalculateServiceName(RunnerSettings settings, string serviceNamePattern, string serviceDisplayNamePattern, out string serviceName, out string serviceDisplayName)
{
Trace.Entering();
serviceName = string.Empty;
serviceDisplayName = string.Empty;
Uri accountUri = new Uri(settings.ServerUrl);
string accountName = string.Empty;
if (accountUri.Host.EndsWith(".githubusercontent.com", StringComparison.OrdinalIgnoreCase))
{
accountName = accountUri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries).FirstOrDefault();
}
else
{
accountName = accountUri.Host.Split('.').FirstOrDefault();
}
if (string.IsNullOrEmpty(accountName))
{
throw new InvalidOperationException($"Cannot find GitHub organization name from server url: '{settings.ServerUrl}'");
}
serviceName = StringUtil.Format(serviceNamePattern, accountName, settings.PoolName, settings.AgentName);
serviceDisplayName = StringUtil.Format(serviceDisplayNamePattern, accountName, settings.PoolName, settings.AgentName);
Trace.Info($"Service name '{serviceName}' display name '{serviceDisplayName}' will be used for service configuration.");
}
}
}

View File

@@ -0,0 +1,55 @@
#if OS_LINUX
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Listener.Configuration
{
public class SystemDControlManager : ServiceControlManager, ILinuxServiceControlManager
{
// This is the name you would see when you do `systemctl list-units | grep runner`
private const string _svcNamePattern = "actions.runner.{0}.{1}.{2}.service";
private const string _svcDisplayPattern = "GitHub Actions Runner ({0}.{1}.{2})";
private const string _shTemplate = "systemd.svc.sh.template";
private const string _shName = "svc.sh";
public void GenerateScripts(RunnerSettings settings)
{
try
{
string serviceName;
string serviceDisplayName;
CalculateServiceName(settings, _svcNamePattern, _svcDisplayPattern, out serviceName, out serviceDisplayName);
string svcShPath = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), _shName);
string svcShContent = File.ReadAllText(Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), _shTemplate));
var tokensToReplace = new Dictionary<string, string>
{
{ "{{SvcDescription}}", serviceDisplayName },
{ "{{SvcNameVar}}", serviceName }
};
svcShContent = tokensToReplace.Aggregate(
svcShContent,
(current, item) => current.Replace(item.Key, item.Value));
File.WriteAllText(svcShPath, svcShContent, new UTF8Encoding(false));
var unixUtil = HostContext.CreateService<IUnixUtil>();
unixUtil.ChmodAsync("755", svcShPath).GetAwaiter().GetResult();
}
catch (Exception ex)
{
Trace.Error(ex);
throw;
}
}
}
}
#endif

Some files were not shown because too many files have changed in this diff Show More