Compare commits

..

11 Commits

Author SHA1 Message Date
John Wesley Walker III
e3e42889af Keep Issue.cs and TimelineRecord.cs aligned in terms of instantiation flow. 2023-02-27 20:48:10 +00:00
John Wesley Walker III
f3961c4895 Refined deserialization fixup. 2023-02-27 17:45:57 +00:00
John Wesley Walker III
cb89be7aac Account for DataContractSerializer vagaries. 2023-02-27 16:38:45 +00:00
John Wesley Walker III
b70f97f183 Restored a using directive that proved to be necessary.
In some cases, references to `System.IO.Compression` members are #ifdef'd and only needed when the OS_WINDOWS compiler directive is present.
2023-02-08 14:15:14 +00:00
John Wesley Walker III
14096a7ee4 Fixed/simplified unit tests.
Close-over KeyValuePairs passed via IssueMetadata to ensure no deferred evaluation.
(An `IEnumerable<KeyValuePair<string, string>>` could very well be a mutable dictionary, so we want to capture its content in the moment -- not some future version of it.)
2023-02-08 13:33:26 +00:00
John Wesley Walker III
4eb9adc958 Added a setter to GitHub.DistributedTask.WebApi.Issue's key-value-pair indexer. 2023-02-07 07:45:00 +00:00
John Wesley Walker III
efc0a92cc7 Let the IssueMetadata class be the mechanism for specifying log message overrides. 2023-01-17 13:43:04 +00:00
John Wesley Walker III
87ababb858 Pivot toward Immutable DistributedTask.WebApi::Issue instances.
The proper way to instantiate a DistributedTask.WebApi::Issue is now via the factory method on an ExecutionContext (IExecutionContext::CreateIssue)
CreateIssue factory.
2023-01-17 11:36:51 +00:00
John Wesley Walker III
3902257e9d Make logging behavior optional in ExecutionContext::AddIssue 2023-01-16 23:34:26 +00:00
Wes Walker
7036627d47 Ensure collected issues are processed only by a non-embedded ExecutionContext.
This was already implicit.  Now, just making it explicit.
2022-12-09 10:37:44 +01:00
Wes Walker
2eeb90a944 [1742] Ensure multiple composite annoations are correctly written.
This implementation uses a collector pattern to allow embedded ExecutionContexts to stash Issue objects for later processing by a non-embedded ancestor ExecutionContext.

Also:
 - Provide explicit constructor implementations for ExecutionContext
 - Leverage explicit constructors to solidify immutability of several ExecutionContext class members.
 - Fixed erroneous call to ExecutionContext.Complete in CompositeActionHandler.cs
 - Use a consistent timestamp for FinishTime in ExecutionContext::Complete
2022-12-09 01:11:21 +01:00
120 changed files with 4781 additions and 17743 deletions

View File

@@ -5,10 +5,7 @@
"features": { "features": {
"ghcr.io/devcontainers/features/docker-in-docker:1": {}, "ghcr.io/devcontainers/features/docker-in-docker:1": {},
"ghcr.io/devcontainers/features/dotnet": { "ghcr.io/devcontainers/features/dotnet": {
"version": "6.0.405" "version": "6.0.300"
},
"ghcr.io/devcontainers/features/node:1": {
"version": "16"
} }
}, },
"customizations": { "customizations": {

View File

@@ -1,8 +1,5 @@
blank_issues_enabled: false blank_issues_enabled: false
contact_links: contact_links:
- name: 🛑 Request a feature in the runner application
url: https://github.com/orgs/community/discussions/categories/actions-and-packages
about: If you have feature requests for GitHub Actions, please use the Actions and Packages section on the Github Product Feedback page.
- name: ✅ Support for GitHub Actions - name: ✅ Support for GitHub Actions
url: https://github.community/c/code-to-cloud/52 url: https://github.community/c/code-to-cloud/52
about: If you have questions about GitHub Actions or need support writing workflows, please ask in the GitHub Community Support forum. about: If you have questions about GitHub Actions or need support writing workflows, please ask in the GitHub Community Support forum.

View File

@@ -0,0 +1,32 @@
---
name: 🛑 Request a feature in the runner application
about: If you have feature requests for GitHub Actions, please use the "feedback and suggestions for GitHub Actions" link below.
title: ''
labels: enhancement
assignees: ''
---
<!--
👋 You're opening a request for an enhancement in the GitHub Actions **runner application**.
🛑 Please stop if you're not certain that the feature you want is in the runner application - if you have a suggestion for improving GitHub Actions, please see the [GitHub Actions Feedback](https://github.com/github/feedback/discussions/categories/actions-and-packages-feedback) discussion forum which is actively monitored. Using the forum ensures that we route your problem to the correct team. 😃
Some additional useful links:
* If you have found a security issue [please submit it here](https://hackerone.com/github)
* If you have questions or issues with the service, writing workflows or actions, then please [visit the GitHub Community Forum's Actions Board](https://github.community/t5/GitHub-Actions/bd-p/actions)
* If you are having an issue or have a question about GitHub Actions then please [contact customer support](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/about-github-actions#contacting-support)
If you have a feature request that is relevant to this repository, the runner, then please include the information below:
-->
**Describe the enhancement**
A clear and concise description of what the features or enhancement you need.
**Code Snippet**
If applicable, add a code snippet.
**Additional information**
Add any other context about the feature here.
NOTE: if the feature request has been agreed upon then the assignee will create an ADR. See docs/adrs/README.md

View File

@@ -27,7 +27,7 @@ jobs:
# Initializes the CodeQL tools for scanning. # Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL - name: Initialize CodeQL
uses: github/codeql-action/init@v2 uses: github/codeql-action/init@v1
# Override language selection by uncommenting this and choosing your languages # Override language selection by uncommenting this and choosing your languages
# with: # with:
# languages: go, javascript, csharp, python, cpp, java # languages: go, javascript, csharp, python, cpp, java
@@ -38,4 +38,4 @@ jobs:
working-directory: src working-directory: src
- name: Perform CodeQL Analysis - name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2 uses: github/codeql-action/analyze@v1

View File

@@ -18,6 +18,7 @@ jobs:
uses: github/super-linter@v4 uses: github/super-linter@v4
env: env:
DEFAULT_BRANCH: ${{ github.base_ref }} DEFAULT_BRANCH: ${{ github.base_ref }}
DISABLE_ERRORS: true
EDITORCONFIG_FILE_NAME: .editorconfig EDITORCONFIG_FILE_NAME: .editorconfig
LINTER_RULES_PATH: /src/ LINTER_RULES_PATH: /src/
VALIDATE_ALL_CODEBASE: false VALIDATE_ALL_CODEBASE: false

View File

@@ -158,11 +158,3 @@ cat (Runner/Worker)_TIMESTAMP.log # view your log file
We use the .NET Foundation and CoreCLR style guidelines [located here]( We use the .NET Foundation and CoreCLR style guidelines [located here](
https://github.com/dotnet/corefx/blob/master/Documentation/coding-guidelines/coding-style.md) https://github.com/dotnet/corefx/blob/master/Documentation/coding-guidelines/coding-style.md)
### Format C# Code
To format both staged and unstaged .cs files
```
cd ./src
./dev.(cmd|sh) format
```

View File

@@ -35,7 +35,7 @@ All the configs below can be found in `.vscode/launch.json`.
If you launch `Run` or `Run [build]`, it starts a process called `Runner.Listener`. If you launch `Run` or `Run [build]`, it starts a process called `Runner.Listener`.
This process will receive any job queued on this repository if the job runs on matching labels (e.g `runs-on: self-hosted`). This process will receive any job queued on this repository if the job runs on matching labels (e.g `runs-on: self-hosted`).
Once a job is received, a `Runner.Listener` starts a new process of `Runner.Worker`. Once a job is received, a `Runner.Listener` starts a new process of `Runner.Worker`.
Since this is a different process, you can't use the same debugger session debug it. Since this is a diferent process, you can't use the same debugger session debug it.
Instead, a parallel debugging session has to be started, using a different launch config. Instead, a parallel debugging session has to be started, using a different launch config.
Luckily, VS Code supports multiple parallel debugging sessions. Luckily, VS Code supports multiple parallel debugging sessions.

View File

@@ -2,8 +2,7 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 as build
ARG RUNNER_VERSION ARG RUNNER_VERSION
ARG RUNNER_ARCH="x64" ARG RUNNER_ARCH="x64"
ARG RUNNER_CONTAINER_HOOKS_VERSION=0.2.0 ARG RUNNER_CONTAINER_HOOKS_VERSION=0.1.3
ARG DOCKER_VERSION=20.10.23
RUN apt update -y && apt install curl unzip -y RUN apt update -y && apt install curl unzip -y
@@ -16,34 +15,10 @@ RUN curl -f -L -o runner-container-hooks.zip https://github.com/actions/runner-c
&& unzip ./runner-container-hooks.zip -d ./k8s \ && unzip ./runner-container-hooks.zip -d ./k8s \
&& rm runner-container-hooks.zip && rm runner-container-hooks.zip
RUN export DOCKER_ARCH=x86_64 \
&& if [ "$RUNNER_ARCH" = "arm64" ]; then export DOCKER_ARCH=aarch64 ; fi \
&& curl -fLo docker.tgz https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-${DOCKER_VERSION}.tgz \
&& tar zxvf docker.tgz \
&& rm -rf docker.tgz
FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 FROM mcr.microsoft.com/dotnet/runtime-deps:6.0
ENV DEBIAN_FRONTEND=noninteractive ENV RUNNER_ALLOW_RUNASROOT=1
ENV RUNNER_MANUALLY_TRAP_SIG=1 ENV RUNNER_MANUALLY_TRAP_SIG=1
ENV ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT=1
RUN apt-get update -y \ WORKDIR /actions-runner
&& apt-get install -y --no-install-recommends \ COPY --from=build /actions-runner .
sudo \
&& rm -rf /var/lib/apt/lists/*
RUN adduser --disabled-password --gecos "" --uid 1001 runner \
&& groupadd docker --gid 123 \
&& usermod -aG sudo runner \
&& usermod -aG docker runner \
&& echo "%sudo ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers \
&& echo "Defaults env_keep += \"DEBIAN_FRONTEND\"" >> /etc/sudoers
WORKDIR /home/runner
COPY --chown=runner:docker --from=build /actions-runner .
RUN install -o root -g root -m 755 docker/* /usr/bin/ && rm -rf docker
USER runner

View File

@@ -1,19 +1,18 @@
## Features ## Features
- Add support for ghe.com domain (#2420) - Displays the error logs in dedicated sub-sections of the Initialize containers section (#2182)
- Add docker cli to the runner image. (#2425) - Add generateServiceConfig option for configure command (#2226)
- Setting debug using GitHub Action variables (#2234)
- run.sh installs SIGINT and SIGTERM traps to gracefully stop runner (#2233, #2240)
## Bugs ## Bugs
- Fix URL construction bug for RunService (#2396) - Use Global.Variables instead of JobContext and include action path/ref in the message. (#2214)
- Defer evaluation of a step's DisplayName until its condition is evaluated. (#2313)
- Replace '(' and ')' with '[' and '] from OS.Description for fixing User-Agent header validation (#2288)
## Misc ## Misc
- Bump dotnet sdk to latest version. (#2392) - Allow '--disableupdate' in create-latest-svc.sh (#2201)
- Start calling run service for job completion (#2412, #2423) - Fix markup for support link (#2114)
- Add runner devcontainer (#2187)
_Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. - Setup linter for Runner (#2211, #2213, #2216)
To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository.
See https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/adding-self-hosted-runners_
## Windows x64 ## Windows x64
We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.

View File

@@ -1 +1 @@
39f2a931565d6a10e695ac8ed14bb9dcbb568151410349b32dbf9c27bae29602 1d709d93e5d3c6c6c656a61aa6c1781050224788a05b0e6ecc4c3c0408bdf89c

View File

@@ -1 +1 @@
29ffb303537d8ba674fbebc7729292c21c4ebd17b3198f91ed593ef4cbbb67b5 b92a47cfeaad02255b1f7a377060651b73ae5e5db22a188dbbcb4183ab03a03d

View File

@@ -1 +1 @@
de6868a836fa3cb9e5ddddbc079da1c25e819aa2d2fc193cc9931c353687c57c 68a9a8ef0843a8bb74241894f6f63fd76241a82295c5337d3cc7a940a314c78e

View File

@@ -1 +1 @@
339d3e1a5fd28450c0fe6cb820cc7aae291f0f9e2d153ac34e1f7b080e35d30e 02c7126ff4d63ee2a0ae390c81434c125630522aadf35903bbeebb1a99d8af99

View File

@@ -1 +1 @@
dcb7f606c1d7d290381e5020ee73e7f16dcbd2f20ac9b431362ccbb5120d449c c9d5a542f8d765168855a89e83ae0a8970d00869041c4f9a766651c04c72b212

View File

@@ -1 +1 @@
1bbcb0e9a2cf4be4b1fce77458de139b70ac58efcbb415a6db028b9373ae1673 39d0683f0f115a211cb10c473e9574c16549a19d4e9a6c637ded3d7022bf809f

View File

@@ -1 +1 @@
44cd25f3c104d0abb44d262397a80e0b2c4f206465c5d899a22eec043dac0fb3 d94f2fbaf210297162bc9f3add819d73682c3aa6899e321c3872412b924d5504

View File

@@ -1 +1 @@
3807dcbf947e840c33535fb466b096d76bf09e5c0254af8fc8cbbb24c6388222 6ed30a2c1ee403a610d63e82bb230b9ba846a9c25cec9e4ea8672fb6ed4e1a51

View File

@@ -1 +1 @@
ee01eee80cd8a460a4b9780ee13fdd20f25c59e754b4ccd99df55fbba2a85634 711c30c51ec52c9b7a9a2eb399d6ab2ab5ee1dc72de11879f2f36f919f163d78

View File

@@ -1 +1 @@
a9fb9c14e24e79aec97d4da197dd7bfc6364297d6fce573afb2df48cc9a931f8 a49479ca4b4988a06c097e8d22c51fd08a11c13f40807366236213d0e008cf6a

View File

@@ -1 +1 @@
a4e0e8fc62eba0967a39c7d693dcd0aeb8b2bed0765f9c38df80d42884f65341 cc4708962a80325de0baa5ae8484e0cb9ae976ac6a4178c1c0d448b8c52bd7f7

View File

@@ -1 +1 @@
17ac17fbe785b3d6fa2868d8d17185ebfe0c90b4b0ddf6b67eac70e42bcd989b 8e97df75230b843462a9b4c578ccec604ee4b4a1066120c85b04374317fa372b

View File

@@ -1 +1 @@
89f24657a550f1e818b0e9975e5b80edcf4dd22b7d4bccbb9e48e37f45d30fb1 e5dace2d41cc0682d096dcce4970079ad48ec7107e46195970eecfdb3df2acef

View File

@@ -1 +1 @@
24fd131b5dce33ef16038b771407bc0507da8682a72fb3b7780607235f76db0b f75a671e5a188c76680739689aa75331a2c09d483dce9c80023518c48fd67a18

View File

@@ -14,7 +14,7 @@
"devDependencies": { "devDependencies": {
"@types/node": "^12.7.12", "@types/node": "^12.7.12",
"@typescript-eslint/parser": "^5.15.0", "@typescript-eslint/parser": "^5.15.0",
"@vercel/ncc": "^0.36.0", "@zeit/ncc": "^0.20.5",
"eslint": "^8.11.0", "eslint": "^8.11.0",
"eslint-plugin-github": "^4.3.5", "eslint-plugin-github": "^4.3.5",
"prettier": "^1.19.1", "prettier": "^1.19.1",
@@ -346,10 +346,11 @@
"url": "https://opencollective.com/typescript-eslint" "url": "https://opencollective.com/typescript-eslint"
} }
}, },
"node_modules/@vercel/ncc": { "node_modules/@zeit/ncc": {
"version": "0.36.0", "version": "0.20.5",
"resolved": "https://registry.npmjs.org/@vercel/ncc/-/ncc-0.36.0.tgz", "resolved": "https://registry.npmjs.org/@zeit/ncc/-/ncc-0.20.5.tgz",
"integrity": "sha512-/ZTUJ/ZkRt694k7KJNimgmHjtQcRuVwsST2Z6XfYveQIuBbHR+EqkTc1jfgPkQmMyk/vtpxo3nVxe8CNuau86A==", "integrity": "sha512-XU6uzwvv95DqxciQx+aOLhbyBx/13ky+RK1y88Age9Du3BlA4mMPCy13BGjayOrrumOzlq1XV3SD/BWiZENXlw==",
"deprecated": "@zeit/ncc is no longer maintained. Please use @vercel/ncc instead.",
"dev": true, "dev": true,
"bin": { "bin": {
"ncc": "dist/ncc/cli.js" "ncc": "dist/ncc/cli.js"
@@ -1721,9 +1722,9 @@
"dev": true "dev": true
}, },
"node_modules/json5": { "node_modules/json5": {
"version": "1.0.2", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz",
"integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==",
"dev": true, "dev": true,
"dependencies": { "dependencies": {
"minimist": "^1.2.0" "minimist": "^1.2.0"
@@ -1823,9 +1824,9 @@
} }
}, },
"node_modules/minimatch": { "node_modules/minimatch": {
"version": "3.1.2", "version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dependencies": { "dependencies": {
"brace-expansion": "^1.1.7" "brace-expansion": "^1.1.7"
}, },
@@ -2746,10 +2747,10 @@
"eslint-visitor-keys": "^3.0.0" "eslint-visitor-keys": "^3.0.0"
} }
}, },
"@vercel/ncc": { "@zeit/ncc": {
"version": "0.36.0", "version": "0.20.5",
"resolved": "https://registry.npmjs.org/@vercel/ncc/-/ncc-0.36.0.tgz", "resolved": "https://registry.npmjs.org/@zeit/ncc/-/ncc-0.20.5.tgz",
"integrity": "sha512-/ZTUJ/ZkRt694k7KJNimgmHjtQcRuVwsST2Z6XfYveQIuBbHR+EqkTc1jfgPkQmMyk/vtpxo3nVxe8CNuau86A==", "integrity": "sha512-XU6uzwvv95DqxciQx+aOLhbyBx/13ky+RK1y88Age9Du3BlA4mMPCy13BGjayOrrumOzlq1XV3SD/BWiZENXlw==",
"dev": true "dev": true
}, },
"acorn": { "acorn": {
@@ -3755,9 +3756,9 @@
"dev": true "dev": true
}, },
"json5": { "json5": {
"version": "1.0.2", "version": "1.0.1",
"resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz",
"integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==",
"dev": true, "dev": true,
"requires": { "requires": {
"minimist": "^1.2.0" "minimist": "^1.2.0"
@@ -3839,9 +3840,9 @@
} }
}, },
"minimatch": { "minimatch": {
"version": "3.1.2", "version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"requires": { "requires": {
"brace-expansion": "^1.1.7" "brace-expansion": "^1.1.7"
} }

View File

@@ -26,7 +26,7 @@
"devDependencies": { "devDependencies": {
"@types/node": "^12.7.12", "@types/node": "^12.7.12",
"@typescript-eslint/parser": "^5.15.0", "@typescript-eslint/parser": "^5.15.0",
"@vercel/ncc": "^0.36.0", "@zeit/ncc": "^0.20.5",
"eslint": "^8.11.0", "eslint": "^8.11.0",
"eslint-plugin-github": "^4.3.5", "eslint-plugin-github": "^4.3.5",
"prettier": "^1.19.1", "prettier": "^1.19.1",

View File

@@ -1,3 +0,0 @@
dist/
lib/
node_modules/

View File

@@ -1,61 +0,0 @@
{
"plugins": ["@typescript-eslint"],
"extends": ["plugin:github/recommended"],
"parser": "@typescript-eslint/parser",
"parserOptions": {
"ecmaVersion": 9,
"sourceType": "module",
"project": "./tsconfig.json"
},
"rules": {
"eslint-comments/no-use": "off",
"import/no-namespace": "off",
"no-console": "off",
"no-unused-vars": "off",
"@typescript-eslint/no-unused-vars": "error",
"@typescript-eslint/explicit-member-accessibility": ["error", {"accessibility": "no-public"}],
"@typescript-eslint/no-require-imports": "error",
"@typescript-eslint/array-type": "error",
"@typescript-eslint/await-thenable": "error",
"@typescript-eslint/naming-convention": [
"error",
{
"selector": "default",
"format": ["camelCase"]
}
],
"camelcase": "off",
"@typescript-eslint/explicit-function-return-type": ["error", {"allowExpressions": true}],
"@typescript-eslint/func-call-spacing": ["error", "never"],
"@typescript-eslint/no-array-constructor": "error",
"@typescript-eslint/no-empty-interface": "error",
"@typescript-eslint/no-explicit-any": "error",
"@typescript-eslint/no-extraneous-class": "error",
"@typescript-eslint/no-for-in-array": "error",
"@typescript-eslint/no-inferrable-types": "error",
"@typescript-eslint/no-misused-new": "error",
"@typescript-eslint/no-namespace": "error",
"@typescript-eslint/no-non-null-assertion": "warn",
"@typescript-eslint/no-unnecessary-qualifier": "error",
"@typescript-eslint/no-unnecessary-type-assertion": "error",
"@typescript-eslint/no-useless-constructor": "error",
"@typescript-eslint/no-var-requires": "error",
"@typescript-eslint/prefer-for-of": "warn",
"@typescript-eslint/prefer-function-type": "warn",
"@typescript-eslint/prefer-includes": "error",
"@typescript-eslint/prefer-string-starts-ends-with": "error",
"@typescript-eslint/promise-function-async": "error",
"@typescript-eslint/require-array-sort-compare": "error",
"@typescript-eslint/restrict-plus-operands": "error",
"@typescript-eslint/semi": ["error", "never"],
"@typescript-eslint/type-annotation-spacing": "error",
"@typescript-eslint/unbound-method": "error",
"filenames/match-regex" : "off",
"github/no-then" : 1, // warning
"semi": "off"
},
"env": {
"node": true,
"es6": true
}
}

View File

@@ -1,3 +0,0 @@
dist/
lib/
node_modules/

View File

@@ -1,11 +0,0 @@
{
"printWidth": 80,
"tabWidth": 2,
"useTabs": false,
"semi": false,
"singleQuote": true,
"trailingComma": "none",
"bracketSpacing": false,
"arrowParens": "avoid",
"parser": "typescript"
}

View File

@@ -1,4 +0,0 @@
To compile this package (output will be stored in `Misc/layoutbin`) run `npm install && npm run all`.
> Note: this package also needs to be recompiled for dependabot PRs updating one of
> its dependencies.

File diff suppressed because it is too large Load Diff

View File

@@ -1,36 +0,0 @@
{
"name": "hashFiles",
"version": "1.0.0",
"description": "GitHub Actions HashFiles() expression function",
"main": "lib/hashFiles.js",
"scripts": {
"build": "tsc",
"format": "prettier --write **/*.ts",
"format-check": "prettier --check **/*.ts",
"lint": "eslint src/**/*.ts",
"pack": "ncc build -o ../../layoutbin/hashFilesV2",
"all": "npm run build && npm run format && npm run lint && npm run pack"
},
"repository": {
"type": "git",
"url": "git+https://github.com/actions/runner.git"
},
"keywords": [
"actions"
],
"author": "GitHub Actions",
"license": "MIT",
"dependencies": {
"@actions/glob": "^0.4.0"
},
"devDependencies": {
"@types/node": "^12.7.12",
"@typescript-eslint/parser": "^5.15.0",
"@vercel/ncc": "^0.36.0",
"eslint": "^8.11.0",
"eslint-plugin-github": "^4.3.5",
"eslint-plugin-prettier": "^4.2.1",
"prettier": "^2.8.4",
"typescript": "^3.6.4"
}
}

View File

@@ -1,31 +0,0 @@
import * as glob from '@actions/glob'
async function run(): Promise<void> {
// arg0 -> node
// arg1 -> hashFiles.js
// env[followSymbolicLinks] = true/null
// env[patterns] -> glob patterns
let followSymbolicLinks = false
const matchPatterns = process.env.patterns || ''
if (process.env.followSymbolicLinks === 'true') {
console.log('Follow symbolic links')
followSymbolicLinks = true
}
const githubWorkspace = process.cwd()
console.log(`Match Pattern: ${matchPatterns}`)
try {
const result = await glob.hashFiles(
matchPatterns,
githubWorkspace,
{followSymbolicLinks},
true
)
console.error(`__OUTPUT__${result}__OUTPUT__`)
process.exit(0)
} catch (error) {
console.log(error)
process.exit(1)
}
}
run()

View File

@@ -1,12 +0,0 @@
{
"compilerOptions": {
"target": "es6", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019' or 'ESNEXT'. */
"module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
"outDir": "./lib", /* Redirect output structure to the directory. */
"rootDir": "./src", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
"strict": true, /* Enable all strict type-checking options. */
"noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */
"esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
},
"exclude": ["node_modules", "**/*.test.ts"]
}

View File

@@ -5,7 +5,7 @@ PRECACHE=$2
NODE_URL=https://nodejs.org/dist NODE_URL=https://nodejs.org/dist
UNOFFICIAL_NODE_URL=https://unofficial-builds.nodejs.org/download/release UNOFFICIAL_NODE_URL=https://unofficial-builds.nodejs.org/download/release
NODE12_VERSION="12.22.7" NODE12_VERSION="12.22.7"
NODE16_VERSION="16.16.0" NODE16_VERSION="16.13.0"
get_abs_path() { get_abs_path() {
# exploits the fact that pwd will print abs path when no args # exploits the fact that pwd will print abs path when no args

View File

@@ -24,8 +24,7 @@ if (exitServiceAfterNFailures <= 0) {
exitServiceAfterNFailures = NaN; exitServiceAfterNFailures = NaN;
} }
var unknownFailureRetryCount = 0; var consecutiveFailureCount = 0;
var retriableFailureRetryCount = 0;
var gracefulShutdown = function () { var gracefulShutdown = function () {
console.log("Shutting down runner listener"); console.log("Shutting down runner listener");
@@ -63,8 +62,7 @@ var runService = function () {
listener.stdout.on("data", (data) => { listener.stdout.on("data", (data) => {
if (data.toString("utf8").includes("Listening for Jobs")) { if (data.toString("utf8").includes("Listening for Jobs")) {
unknownFailureRetryCount = 0; consecutiveFailureCount = 0;
retriableFailureRetryCount = 0;
} }
process.stdout.write(data.toString("utf8")); process.stdout.write(data.toString("utf8"));
}); });
@@ -94,38 +92,24 @@ var runService = function () {
console.log( console.log(
"Runner listener exit with retryable error, re-launch runner in 5 seconds." "Runner listener exit with retryable error, re-launch runner in 5 seconds."
); );
unknownFailureRetryCount = 0; consecutiveFailureCount = 0;
retriableFailureRetryCount++;
if (retriableFailureRetryCount >= 10) {
console.error(
"Stopping the runner after 10 consecutive re-tryable failures"
);
stopping = true;
}
} else if (code === 3 || code === 4) { } else if (code === 3 || code === 4) {
console.log( console.log(
"Runner listener exit because of updating, re-launch runner in 5 seconds." "Runner listener exit because of updating, re-launch runner in 5 seconds."
); );
unknownFailureRetryCount = 0; consecutiveFailureCount = 0;
retriableFailureRetryCount++;
if (retriableFailureRetryCount >= 10) {
console.error(
"Stopping the runner after 10 consecutive re-tryable failures"
);
stopping = true;
}
} else { } else {
var messagePrefix = "Runner listener exit with undefined return code"; var messagePrefix = "Runner listener exit with undefined return code";
unknownFailureRetryCount++; consecutiveFailureCount++;
retriableFailureRetryCount = 0;
if ( if (
!isNaN(exitServiceAfterNFailures) && !isNaN(exitServiceAfterNFailures) &&
unknownFailureRetryCount >= exitServiceAfterNFailures consecutiveFailureCount >= exitServiceAfterNFailures
) { ) {
console.error( console.error(
`${messagePrefix}, exiting service after ${unknownFailureRetryCount} consecutive failures` `${messagePrefix}, exiting service after ${consecutiveFailureCount} consecutive failures`
); );
stopping = true gracefulShutdown();
return;
} else { } else {
console.log(`${messagePrefix}, re-launch runner in 5 seconds.`); console.log(`${messagePrefix}, re-launch runner in 5 seconds.`);
} }

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -18,20 +18,6 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
done done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
# Wait for docker to start
if [ ! -z "$RUNNER_WAIT_FOR_DOCKER_IN_SECONDS" ]; then
if [ "$RUNNER_WAIT_FOR_DOCKER_IN_SECONDS" -gt 0 ]; then
echo "Waiting for docker to be ready."
for i in $(seq "$RUNNER_WAIT_FOR_DOCKER_IN_SECONDS"); do
if docker ps > /dev/null 2>&1; then
echo "Docker is ready."
break
fi
"$DIR"/safe_sleep.sh 1
done
fi
fi
updateFile="update.finished" updateFile="update.finished"
"$DIR"/bin/Runner.Listener run $* "$DIR"/bin/Runner.Listener run $*

View File

@@ -4,7 +4,6 @@ checkScripts/downloadCert.js
checkScripts/makeWebRequest.js checkScripts/makeWebRequest.js
darwin.svc.sh.template darwin.svc.sh.template
hashFiles/index.js hashFiles/index.js
hashFilesV2/index.js
installdependencies.sh installdependencies.sh
macos-run-invoker.js macos-run-invoker.js
Microsoft.IdentityModel.Logging.dll Microsoft.IdentityModel.Logging.dll

View File

@@ -74,7 +74,6 @@ Microsoft.Win32.Registry.dll
mscordaccore.dll mscordaccore.dll
mscordaccore_amd64_amd64_6.0.522.21309.dll mscordaccore_amd64_amd64_6.0.522.21309.dll
mscordaccore_arm64_arm64_6.0.522.21309.dll mscordaccore_arm64_arm64_6.0.522.21309.dll
mscordaccore_amd64_amd64_6.0.1322.58009.dll
mscordbi.dll mscordbi.dll
mscorlib.dll mscorlib.dll
mscorrc.debug.dll mscorrc.debug.dll

View File

@@ -1,4 +1,4 @@
using System; using System;
namespace GitHub.Runner.Common namespace GitHub.Runner.Common
{ {
@@ -90,6 +90,7 @@ namespace GitHub.Runner.Common
public static class Args public static class Args
{ {
public static readonly string Auth = "auth"; public static readonly string Auth = "auth";
public static readonly string JitConfig = "jitconfig";
public static readonly string Labels = "labels"; public static readonly string Labels = "labels";
public static readonly string MonitorSocketAddress = "monitorsocketaddress"; public static readonly string MonitorSocketAddress = "monitorsocketaddress";
public static readonly string Name = "name"; public static readonly string Name = "name";
@@ -104,13 +105,11 @@ namespace GitHub.Runner.Common
public static readonly string Token = "token"; public static readonly string Token = "token";
public static readonly string PAT = "pat"; public static readonly string PAT = "pat";
public static readonly string WindowsLogonPassword = "windowslogonpassword"; public static readonly string WindowsLogonPassword = "windowslogonpassword";
public static readonly string JitConfig = "jitconfig";
public static string[] Secrets => new[] public static string[] Secrets => new[]
{ {
PAT, PAT,
Token, Token,
WindowsLogonPassword, WindowsLogonPassword,
JitConfig,
}; };
} }
@@ -131,7 +130,6 @@ namespace GitHub.Runner.Common
public static readonly string Ephemeral = "ephemeral"; public static readonly string Ephemeral = "ephemeral";
public static readonly string GenerateServiceConfig = "generateServiceConfig"; public static readonly string GenerateServiceConfig = "generateServiceConfig";
public static readonly string Help = "help"; public static readonly string Help = "help";
public static readonly string Local = "local";
public static readonly string Replace = "replace"; public static readonly string Replace = "replace";
public static readonly string DisableUpdate = "disableupdate"; public static readonly string DisableUpdate = "disableupdate";
public static readonly string Once = "once"; // Keep this around since customers still relies on it public static readonly string Once = "once"; // Keep this around since customers still relies on it
@@ -156,20 +154,16 @@ namespace GitHub.Runner.Common
public static readonly string Node12Warning = "DistributedTask.AddWarningToNode12Action"; public static readonly string Node12Warning = "DistributedTask.AddWarningToNode12Action";
public static readonly string UseContainerPathForTemplate = "DistributedTask.UseContainerPathForTemplate"; public static readonly string UseContainerPathForTemplate = "DistributedTask.UseContainerPathForTemplate";
public static readonly string AllowRunnerContainerHooks = "DistributedTask.AllowRunnerContainerHooks"; public static readonly string AllowRunnerContainerHooks = "DistributedTask.AllowRunnerContainerHooks";
public static readonly string UseGlobHashFiles = "DistributedTask.UseGlobHashFiles";
} }
public static readonly string InternalTelemetryIssueDataKey = "_internal_telemetry"; public static readonly string InternalTelemetryIssueDataKey = "_internal_telemetry";
public static readonly Guid TelemetryRecordId = new Guid("11111111-1111-1111-1111-111111111111");
public static readonly string WorkerCrash = "WORKER_CRASH"; public static readonly string WorkerCrash = "WORKER_CRASH";
public static readonly string LowDiskSpace = "LOW_DISK_SPACE"; public static readonly string LowDiskSpace = "LOW_DISK_SPACE";
public static readonly string UnsupportedCommand = "UNSUPPORTED_COMMAND"; public static readonly string UnsupportedCommand = "UNSUPPORTED_COMMAND";
public static readonly string ResultsUploadFailure = "RESULTS_UPLOAD_FAILURE";
public static readonly string UnsupportedCommandMessage = "The `{0}` command is deprecated and will be disabled soon. Please upgrade to using Environment Files. For more information see: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/"; public static readonly string UnsupportedCommandMessage = "The `{0}` command is deprecated and will be disabled soon. Please upgrade to using Environment Files. For more information see: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/";
public static readonly string UnsupportedCommandMessageDisabled = "The `{0}` command is disabled. Please upgrade to using Environment Files or opt into unsecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_COMMANDS` environment variable to `true`. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/"; public static readonly string UnsupportedCommandMessageDisabled = "The `{0}` command is disabled. Please upgrade to using Environment Files or opt into unsecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_COMMANDS` environment variable to `true`. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/";
public static readonly string UnsupportedStopCommandTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`."; public static readonly string UnsupportedStopCommandTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`.";
public static readonly string UnsupportedSummarySize = "$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of {0}k, got {1}k. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary"; public static readonly string UnsupportedSummarySize = "$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of {0}k, got {1}k. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary";
public static readonly string SummaryUploadError = "$GITHUB_STEP_SUMMARY upload aborted, an error occurred when uploading the summary. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary";
public static readonly string Node12DetectedAfterEndOfLife = "Node.js 12 actions are deprecated. Please update the following actions to use Node.js 16: {0}. For more information see: https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/."; public static readonly string Node12DetectedAfterEndOfLife = "Node.js 12 actions are deprecated. Please update the following actions to use Node.js 16: {0}. For more information see: https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/.";
} }

View File

@@ -226,20 +226,6 @@ namespace GitHub.Runner.Common
} }
_userAgents.Add(new ProductInfoHeaderValue("CommitSHA", BuildConstants.Source.CommitHash)); _userAgents.Add(new ProductInfoHeaderValue("CommitSHA", BuildConstants.Source.CommitHash));
var extraUserAgent = Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_EXTRA_USER_AGENT");
if (!string.IsNullOrEmpty(extraUserAgent))
{
var extraUserAgentSplit = extraUserAgent.Split('/', StringSplitOptions.RemoveEmptyEntries);
if (extraUserAgentSplit.Length != 2)
{
_trace.Error($"GITHUB_ACTIONS_RUNNER_EXTRA_USER_AGENT is not in the format of 'name/version'.");
}
var extraUserAgentHeader = new ProductInfoHeaderValue(extraUserAgentSplit[0], extraUserAgentSplit[1]);
_trace.Info($"Adding extra user agent '{extraUserAgentHeader}' to all HTTP requests.");
_userAgents.Add(extraUserAgentHeader);
}
} }
public string GetDirectory(WellKnownDirectory directory) public string GetDirectory(WellKnownDirectory directory)

View File

@@ -13,8 +13,6 @@ using GitHub.Runner.Sdk;
using GitHub.Services.Common; using GitHub.Services.Common;
using GitHub.Services.WebApi; using GitHub.Services.WebApi;
using GitHub.Services.WebApi.Utilities.Internal; using GitHub.Services.WebApi.Utilities.Internal;
using GitHub.Services.Results.Client;
using GitHub.Services.OAuth;
namespace GitHub.Runner.Common namespace GitHub.Runner.Common
{ {
@@ -24,15 +22,11 @@ namespace GitHub.Runner.Common
Task ConnectAsync(VssConnection jobConnection); Task ConnectAsync(VssConnection jobConnection);
void InitializeWebsocketClient(ServiceEndpoint serviceEndpoint); void InitializeWebsocketClient(ServiceEndpoint serviceEndpoint);
void InitializeResultsClient(Uri uri, string token);
// logging and console // logging and console
Task<TaskLog> AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken); Task<TaskLog> AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken);
Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long? startLine, CancellationToken cancellationToken); Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long? startLine, CancellationToken cancellationToken);
Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken); Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken);
Task CreateStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken);
Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken);
Task CreateResultsJobLogAsync(string planId, string jobId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken);
Task<TaskLog> CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken); Task<TaskLog> CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken);
Task<Timeline> CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken); Task<Timeline> CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken);
Task<List<TimelineRecord>> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable<TimelineRecord> records, CancellationToken cancellationToken); Task<List<TimelineRecord>> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable<TimelineRecord> records, CancellationToken cancellationToken);
@@ -46,7 +40,6 @@ namespace GitHub.Runner.Common
private bool _hasConnection; private bool _hasConnection;
private VssConnection _connection; private VssConnection _connection;
private TaskHttpClient _taskClient; private TaskHttpClient _taskClient;
private ResultsHttpClient _resultsClient;
private ClientWebSocket _websocketClient; private ClientWebSocket _websocketClient;
private ServiceEndpoint _serviceEndpoint; private ServiceEndpoint _serviceEndpoint;
@@ -150,12 +143,6 @@ namespace GitHub.Runner.Common
InitializeWebsocketClient(TimeSpan.Zero); InitializeWebsocketClient(TimeSpan.Zero);
} }
public void InitializeResultsClient(Uri uri, string token)
{
var httpMessageHandler = HostContext.CreateHttpClientHandler();
this._resultsClient = new ResultsHttpClient(uri, httpMessageHandler, token, disposeHandler: true);
}
public ValueTask DisposeAsync() public ValueTask DisposeAsync()
{ {
CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None); CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None);
@@ -318,33 +305,6 @@ namespace GitHub.Runner.Common
return _taskClient.CreateAttachmentAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, type, name, uploadStream, cancellationToken: cancellationToken); return _taskClient.CreateAttachmentAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, type, name, uploadStream, cancellationToken: cancellationToken);
} }
public Task CreateStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken)
{
if (_resultsClient != null)
{
return _resultsClient.UploadStepSummaryAsync(planId, jobId, stepId, file, cancellationToken: cancellationToken);
}
throw new InvalidOperationException("Results client is not initialized.");
}
public Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken)
{
if (_resultsClient != null)
{
return _resultsClient.UploadResultsStepLogAsync(planId, jobId, stepId, file, finalize, firstBlock, lineCount, cancellationToken: cancellationToken);
}
throw new InvalidOperationException("Results client is not initialized.");
}
public Task CreateResultsJobLogAsync(string planId, string jobId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken)
{
if (_resultsClient != null)
{
return _resultsClient.UploadResultsJobLogAsync(planId, jobId, file, finalize, firstBlock, lineCount, cancellationToken: cancellationToken);
}
throw new InvalidOperationException("Results client is not initialized.");
}
public Task<TaskLog> CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken) public Task<TaskLog> CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken)
{ {
CheckConnection(); CheckConnection();

View File

@@ -20,7 +20,6 @@ namespace GitHub.Runner.Common
void Start(Pipelines.AgentJobRequestMessage jobRequest); void Start(Pipelines.AgentJobRequestMessage jobRequest);
void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null); void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null);
void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource); void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource);
void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines);
void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord); void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord);
} }
@@ -31,7 +30,6 @@ namespace GitHub.Runner.Common
private static readonly TimeSpan _delayForWebConsoleLineDequeue = TimeSpan.FromMilliseconds(500); private static readonly TimeSpan _delayForWebConsoleLineDequeue = TimeSpan.FromMilliseconds(500);
private static readonly TimeSpan _delayForTimelineUpdateDequeue = TimeSpan.FromMilliseconds(500); private static readonly TimeSpan _delayForTimelineUpdateDequeue = TimeSpan.FromMilliseconds(500);
private static readonly TimeSpan _delayForFileUploadDequeue = TimeSpan.FromMilliseconds(1000); private static readonly TimeSpan _delayForFileUploadDequeue = TimeSpan.FromMilliseconds(1000);
private static readonly TimeSpan _delayForResultsUploadDequeue = TimeSpan.FromMilliseconds(1000);
// Job message information // Job message information
private Guid _scopeIdentifier; private Guid _scopeIdentifier;
@@ -46,8 +44,6 @@ namespace GitHub.Runner.Common
// queue for file upload (log file or attachment) // queue for file upload (log file or attachment)
private readonly ConcurrentQueue<UploadFileInfo> _fileUploadQueue = new(); private readonly ConcurrentQueue<UploadFileInfo> _fileUploadQueue = new();
private readonly ConcurrentQueue<ResultsUploadFileInfo> _resultsFileUploadQueue = new();
// queue for timeline or timeline record update (one queue per timeline) // queue for timeline or timeline record update (one queue per timeline)
private readonly ConcurrentDictionary<Guid, ConcurrentQueue<TimelineRecord>> _timelineUpdateQueue = new(); private readonly ConcurrentDictionary<Guid, ConcurrentQueue<TimelineRecord>> _timelineUpdateQueue = new();
@@ -60,7 +56,6 @@ namespace GitHub.Runner.Common
// Task for each queue's dequeue process // Task for each queue's dequeue process
private Task _webConsoleLineDequeueTask; private Task _webConsoleLineDequeueTask;
private Task _fileUploadDequeueTask; private Task _fileUploadDequeueTask;
private Task _resultsUploadDequeueTask;
private Task _timelineUpdateDequeueTask; private Task _timelineUpdateDequeueTask;
// common // common
@@ -84,9 +79,6 @@ namespace GitHub.Runner.Common
private bool _webConsoleLineAggressiveDequeue = true; private bool _webConsoleLineAggressiveDequeue = true;
private bool _firstConsoleOutputs = true; private bool _firstConsoleOutputs = true;
private bool _resultsClientInitiated = false;
private delegate Task ResultsFileUploadHandler(ResultsUploadFileInfo file);
public override void Initialize(IHostContext hostContext) public override void Initialize(IHostContext hostContext)
{ {
base.Initialize(hostContext); base.Initialize(hostContext);
@@ -101,20 +93,6 @@ namespace GitHub.Runner.Common
_jobServer.InitializeWebsocketClient(serviceEndPoint); _jobServer.InitializeWebsocketClient(serviceEndPoint);
// This code is usually wrapped by an instance of IExecutionContext which isn't available here.
jobRequest.Variables.TryGetValue("system.github.results_endpoint", out VariableValue resultsEndpointVariable);
var resultsReceiverEndpoint = resultsEndpointVariable?.Value;
if (serviceEndPoint?.Authorization != null &&
serviceEndPoint.Authorization.Parameters.TryGetValue("AccessToken", out var accessToken) &&
!string.IsNullOrEmpty(accessToken) &&
!string.IsNullOrEmpty(resultsReceiverEndpoint))
{
Trace.Info("Initializing results client");
_jobServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), accessToken);
_resultsClientInitiated = true;
}
if (_queueInProcess) if (_queueInProcess)
{ {
Trace.Info("No-opt, all queue process tasks are running."); Trace.Info("No-opt, all queue process tasks are running.");
@@ -142,13 +120,10 @@ namespace GitHub.Runner.Common
Trace.Info("Start process file upload queue."); Trace.Info("Start process file upload queue.");
_fileUploadDequeueTask = ProcessFilesUploadQueueAsync(); _fileUploadDequeueTask = ProcessFilesUploadQueueAsync();
Trace.Info("Start results file upload queue.");
_resultsUploadDequeueTask = ProcessResultsUploadQueueAsync();
Trace.Info("Start process timeline update queue."); Trace.Info("Start process timeline update queue.");
_timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync(); _timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync();
_allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask, _resultsUploadDequeueTask }; _allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask };
_queueInProcess = true; _queueInProcess = true;
} }
@@ -179,10 +154,6 @@ namespace GitHub.Runner.Common
await ProcessFilesUploadQueueAsync(runOnce: true); await ProcessFilesUploadQueueAsync(runOnce: true);
Trace.Info("File upload queue drained."); Trace.Info("File upload queue drained.");
Trace.Verbose("Draining results upload queue.");
await ProcessResultsUploadQueueAsync(runOnce: true);
Trace.Info("Results upload queue drained.");
// ProcessTimelinesUpdateQueueAsync() will throw exception during shutdown // ProcessTimelinesUpdateQueueAsync() will throw exception during shutdown
// if there is any timeline records that failed to update contains output variabls. // if there is any timeline records that failed to update contains output variabls.
Trace.Verbose("Draining timeline update queue."); Trace.Verbose("Draining timeline update queue.");
@@ -233,45 +204,6 @@ namespace GitHub.Runner.Common
_fileUploadQueue.Enqueue(newFile); _fileUploadQueue.Enqueue(newFile);
} }
public void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines)
{
if (!_resultsClientInitiated)
{
Trace.Verbose("Skipping results upload");
try
{
if (deleteSource)
{
File.Delete(path);
}
}
catch (Exception ex)
{
Trace.Info("Catch exception during delete skipped results upload file.");
Trace.Error(ex);
}
return;
}
// all parameter not null, file path exist.
var newFile = new ResultsUploadFileInfo()
{
Name = name,
Path = path,
Type = type,
PlanId = _planId.ToString(),
JobId = _jobTimelineRecordId.ToString(),
RecordId = timelineRecordId,
DeleteSource = deleteSource,
Finalize = finalize,
FirstBlock = firstBlock,
TotalLines = totalLines,
};
Trace.Verbose("Enqueue results file upload queue: file '{0}' attach to job {1} step {2}", newFile.Path, _jobTimelineRecordId, timelineRecordId);
_resultsFileUploadQueue.Enqueue(newFile);
}
public void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord) public void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord)
{ {
ArgUtil.NotEmpty(timelineId, nameof(timelineId)); ArgUtil.NotEmpty(timelineId, nameof(timelineId));
@@ -462,86 +394,6 @@ namespace GitHub.Runner.Common
} }
} }
private async Task ProcessResultsUploadQueueAsync(bool runOnce = false)
{
Trace.Info("Starting results-based upload queue...");
while (!_jobCompletionSource.Task.IsCompleted || runOnce)
{
List<ResultsUploadFileInfo> filesToUpload = new();
ResultsUploadFileInfo dequeueFile;
while (_resultsFileUploadQueue.TryDequeue(out dequeueFile))
{
filesToUpload.Add(dequeueFile);
// process at most 10 file uploads.
if (!runOnce && filesToUpload.Count > 10)
{
break;
}
}
if (filesToUpload.Count > 0)
{
if (runOnce)
{
Trace.Info($"Uploading {filesToUpload.Count} file(s) in one shot through results service.");
}
int errorCount = 0;
foreach (var file in filesToUpload)
{
try
{
if (String.Equals(file.Type, ChecksAttachmentType.StepSummary, StringComparison.OrdinalIgnoreCase))
{
await UploadSummaryFile(file);
}
else if (String.Equals(file.Type, CoreAttachmentType.ResultsLog, StringComparison.OrdinalIgnoreCase))
{
if (file.RecordId != _jobTimelineRecordId)
{
Trace.Info($"Got a step log file to send to results service.");
await UploadResultsStepLogFile(file);
}
else if (file.RecordId == _jobTimelineRecordId)
{
Trace.Info($"Got a job log file to send to results service.");
await UploadResultsJobLogFile(file);
}
}
}
catch (Exception ex)
{
var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception during file upload to results. {ex.Message}" };
issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.ResultsUploadFailure;
var telemetryRecord = new TimelineRecord()
{
Id = Constants.Runner.TelemetryRecordId,
};
telemetryRecord.Issues.Add(issue);
QueueTimelineRecordUpdate(_jobTimelineId, telemetryRecord);
Trace.Info("Catch exception during file upload to results, keep going since the process is best effort.");
Trace.Error(ex);
errorCount++;
}
}
Trace.Info("Tried to upload {0} file(s) to results, success rate: {1}/{0}.", filesToUpload.Count, filesToUpload.Count - errorCount);
}
if (runOnce)
{
break;
}
else
{
await Task.Delay(_delayForResultsUploadDequeue);
}
}
}
private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false)
{ {
while (!_jobCompletionSource.Task.IsCompleted || runOnce) while (!_jobCompletionSource.Task.IsCompleted || runOnce)
@@ -748,8 +600,7 @@ namespace GitHub.Runner.Common
{ {
foreach (var issue in record.Issues) foreach (var issue in record.Issues)
{ {
String source; string source = issue["sourcepath"];
issue.Data.TryGetValue("sourcepath", out source);
Trace.Verbose($" Issue: c={issue.Category}, t={issue.Type}, s={source ?? string.Empty}, m={issue.Message}"); Trace.Verbose($" Issue: c={issue.Category}, t={issue.Type}, s={source ?? string.Empty}, m={issue.Message}");
} }
} }
@@ -813,64 +664,6 @@ namespace GitHub.Runner.Common
} }
} }
} }
private async Task UploadSummaryFile(ResultsUploadFileInfo file)
{
Trace.Info($"Starting to upload summary file to results service {file.Name}, {file.Path}");
ResultsFileUploadHandler summaryHandler = async (file) =>
{
await _jobServer.CreateStepSummaryAsync(file.PlanId, file.JobId, file.RecordId, file.Path, CancellationToken.None);
};
await UploadResultsFile(file, summaryHandler);
}
private async Task UploadResultsStepLogFile(ResultsUploadFileInfo file)
{
Trace.Info($"Starting upload of step log file to results service {file.Name}, {file.Path}");
ResultsFileUploadHandler stepLogHandler = async (file) =>
{
await _jobServer.CreateResultsStepLogAsync(file.PlanId, file.JobId, file.RecordId, file.Path, file.Finalize, file.FirstBlock, file.TotalLines, CancellationToken.None);
};
await UploadResultsFile(file, stepLogHandler);
}
private async Task UploadResultsJobLogFile(ResultsUploadFileInfo file)
{
Trace.Info($"Starting upload of job log file to results service {file.Name}, {file.Path}");
ResultsFileUploadHandler jobLogHandler = async (file) =>
{
await _jobServer.CreateResultsJobLogAsync(file.PlanId, file.JobId, file.Path, file.Finalize, file.FirstBlock, file.TotalLines, CancellationToken.None);
};
await UploadResultsFile(file, jobLogHandler);
}
private async Task UploadResultsFile(ResultsUploadFileInfo file, ResultsFileUploadHandler uploadHandler)
{
bool uploadSucceed = false;
try
{
await uploadHandler(file);
uploadSucceed = true;
}
finally
{
if (uploadSucceed && file.DeleteSource)
{
try
{
File.Delete(file.Path);
}
catch (Exception ex)
{
Trace.Info("Exception encountered during deletion of a temporary file that was already successfully uploaded to results.");
Trace.Error(ex);
}
}
}
}
} }
internal class PendingTimelineRecord internal class PendingTimelineRecord
@@ -889,21 +682,6 @@ namespace GitHub.Runner.Common
public bool DeleteSource { get; set; } public bool DeleteSource { get; set; }
} }
internal class ResultsUploadFileInfo
{
public string Name { get; set; }
public string Type { get; set; }
public string Path { get; set; }
public string PlanId { get; set; }
public string JobId { get; set; }
public Guid RecordId { get; set; }
public bool DeleteSource { get; set; }
public bool Finalize { get; set; }
public bool FirstBlock { get; set; }
public long TotalLines { get; set; }
}
internal class ConsoleLineInfo internal class ConsoleLineInfo
{ {

View File

@@ -21,12 +21,6 @@ namespace GitHub.Runner.Common
// 8 MB // 8 MB
public const int PageSize = 8 * 1024 * 1024; public const int PageSize = 8 * 1024 * 1024;
// For Results
public static string BlocksFolder = "blocks";
// 2 MB
public const int BlockSize = 2 * 1024 * 1024;
private Guid _timelineId; private Guid _timelineId;
private Guid _timelineRecordId; private Guid _timelineRecordId;
private FileStream _pageData; private FileStream _pageData;
@@ -38,13 +32,6 @@ namespace GitHub.Runner.Common
private string _pagesFolder; private string _pagesFolder;
private IJobServerQueue _jobServerQueue; private IJobServerQueue _jobServerQueue;
private string _resultsDataFileName;
private FileStream _resultsBlockData;
private StreamWriter _resultsBlockWriter;
private string _resultsBlockFolder;
private int _blockByteCount;
private int _blockCount;
public long TotalLines => _totalLines; public long TotalLines => _totalLines;
public override void Initialize(IHostContext hostContext) public override void Initialize(IHostContext hostContext)
@@ -52,10 +39,8 @@ namespace GitHub.Runner.Common
base.Initialize(hostContext); base.Initialize(hostContext);
_totalLines = 0; _totalLines = 0;
_pagesFolder = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Diag), PagingFolder); _pagesFolder = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Diag), PagingFolder);
Directory.CreateDirectory(_pagesFolder);
_resultsBlockFolder = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Diag), BlocksFolder);
Directory.CreateDirectory(_resultsBlockFolder);
_jobServerQueue = HostContext.GetService<IJobServerQueue>(); _jobServerQueue = HostContext.GetService<IJobServerQueue>();
Directory.CreateDirectory(_pagesFolder);
} }
public void Setup(Guid timelineId, Guid timelineRecordId) public void Setup(Guid timelineId, Guid timelineRecordId)
@@ -75,17 +60,11 @@ namespace GitHub.Runner.Common
// lazy creation on write // lazy creation on write
if (_pageWriter == null) if (_pageWriter == null)
{ {
NewPage(); Create();
}
if (_resultsBlockWriter == null)
{
NewBlock();
} }
string line = $"{DateTime.UtcNow.ToString("O")} {message}"; string line = $"{DateTime.UtcNow.ToString("O")} {message}";
_pageWriter.WriteLine(line); _pageWriter.WriteLine(line);
_resultsBlockWriter.WriteLine(line);
_totalLines++; _totalLines++;
if (line.IndexOf('\n') != -1) if (line.IndexOf('\n') != -1)
@@ -99,25 +78,21 @@ namespace GitHub.Runner.Common
} }
} }
var bytes = System.Text.Encoding.UTF8.GetByteCount(line); _byteCount += System.Text.Encoding.UTF8.GetByteCount(line);
_byteCount += bytes;
_blockByteCount += bytes;
if (_byteCount >= PageSize) if (_byteCount >= PageSize)
{ {
NewPage(); NewPage();
} }
if (_blockByteCount >= BlockSize)
{
NewBlock();
}
} }
public void End() public void End()
{ {
EndPage(); EndPage();
EndBlock(true); }
private void Create()
{
NewPage();
} }
private void NewPage() private void NewPage()
@@ -142,27 +117,5 @@ namespace GitHub.Runner.Common
_jobServerQueue.QueueFileUpload(_timelineId, _timelineRecordId, "DistributedTask.Core.Log", "CustomToolLog", _dataFileName, true); _jobServerQueue.QueueFileUpload(_timelineId, _timelineRecordId, "DistributedTask.Core.Log", "CustomToolLog", _dataFileName, true);
} }
} }
private void NewBlock()
{
EndBlock(false);
_blockByteCount = 0;
_resultsDataFileName = Path.Combine(_resultsBlockFolder, $"{_timelineId}_{_timelineRecordId}.{++_blockCount}");
_resultsBlockData = new FileStream(_resultsDataFileName, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite);
_resultsBlockWriter = new StreamWriter(_resultsBlockData, System.Text.Encoding.UTF8);
}
private void EndBlock(bool finalize)
{
if (_resultsBlockWriter != null)
{
_resultsBlockWriter.Flush();
_resultsBlockData.Flush();
_resultsBlockWriter.Dispose();
_resultsBlockWriter = null;
_resultsBlockData = null;
_jobServerQueue.QueueResultsUpload(_timelineRecordId, "ResultsLog", _resultsDataFileName, "Results.Core.Log", deleteSource: true, finalize, firstBlock: _resultsDataFileName.EndsWith(".1"), totalLines: _totalLines);
}
}
} }
} }

View File

@@ -1,13 +1,11 @@
using System; using System;
using System.Collections.Generic;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.Actions.RunService.WebApi;
using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.Pipelines;
using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;
using GitHub.Services.Common; using GitHub.Services.Common;
using Sdk.RSWebApi.Contracts; using GitHub.Services.WebApi;
using Sdk.WebApi.WebApi.RawClient; using Sdk.WebApi.WebApi.RawClient;
namespace GitHub.Runner.Common namespace GitHub.Runner.Common
@@ -18,10 +16,6 @@ namespace GitHub.Runner.Common
Task ConnectAsync(Uri serverUrl, VssCredentials credentials); Task ConnectAsync(Uri serverUrl, VssCredentials credentials);
Task<AgentJobRequestMessage> GetJobMessageAsync(string id, CancellationToken token); Task<AgentJobRequestMessage> GetJobMessageAsync(string id, CancellationToken token);
Task CompleteJobAsync(Guid planId, Guid jobId, TaskResult result, Dictionary<String, VariableValue> outputs, IList<StepResult> stepResults, CancellationToken token);
Task<RenewJobResponse> RenewJobAsync(Guid planId, Guid jobId, CancellationToken token);
} }
public sealed class RunServer : RunnerService, IRunServer public sealed class RunServer : RunnerService, IRunServer
@@ -35,7 +29,7 @@ namespace GitHub.Runner.Common
{ {
requestUri = serverUri; requestUri = serverUri;
_connection = VssUtil.CreateRawConnection(serverUri, credentials); _connection = VssUtil.CreateRawConnection(new Uri(serverUri.Authority), credentials);
_runServiceHttpClient = await _connection.GetClientAsync<RunServiceHttpClient>(); _runServiceHttpClient = await _connection.GetClientAsync<RunServiceHttpClient>();
_hasConnection = true; _hasConnection = true;
} }
@@ -61,24 +55,5 @@ namespace GitHub.Runner.Common
return jobMessage; return jobMessage;
} }
public Task CompleteJobAsync(Guid planId, Guid jobId, TaskResult result, Dictionary<String, VariableValue> outputs, IList<StepResult> stepResults, CancellationToken cancellationToken)
{
CheckConnection();
return RetryRequest(
async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, cancellationToken), cancellationToken);
}
public Task<RenewJobResponse> RenewJobAsync(Guid planId, Guid jobId, CancellationToken cancellationToken)
{
CheckConnection();
var renewJobResponse = RetryRequest<RenewJobResponse>(
async () => await _runServiceHttpClient.RenewJobAsync(requestUri, planId, jobId, cancellationToken), cancellationToken);
if (renewJobResponse == null)
{
throw new TaskOrchestrationJobNotFoundException(jobId.ToString());
}
return renewJobResponse;
}
} }
} }

View File

@@ -68,19 +68,6 @@ namespace GitHub.Runner.Common
throw new InvalidOperationException(nameof(EstablishVssConnection)); throw new InvalidOperationException(nameof(EstablishVssConnection));
} }
protected async Task RetryRequest(Func<Task> func,
CancellationToken cancellationToken,
int maxRetryAttemptsCount = 5
)
{
async Task<Unit> wrappedFunc()
{
await func();
return Unit.Value;
}
await RetryRequest<Unit>(wrappedFunc, cancellationToken, maxRetryAttemptsCount);
}
protected async Task<T> RetryRequest<T>(Func<Task<T>> func, protected async Task<T> RetryRequest<T>(Func<Task<T>> func,
CancellationToken cancellationToken, CancellationToken cancellationToken,
int maxRetryAttemptsCount = 5 int maxRetryAttemptsCount = 5
@@ -98,7 +85,7 @@ namespace GitHub.Runner.Common
// TODO: Add handling of non-retriable exceptions: https://github.com/github/actions-broker/issues/122 // TODO: Add handling of non-retriable exceptions: https://github.com/github/actions-broker/issues/122
catch (Exception ex) when (retryCount < maxRetryAttemptsCount) catch (Exception ex) when (retryCount < maxRetryAttemptsCount)
{ {
Trace.Error("Catch exception during request"); Trace.Error("Catch exception during get full job message");
Trace.Error(ex); Trace.Error(ex);
var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)); var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
Trace.Warning($"Back off {backOff.TotalSeconds} seconds before next retry. {maxRetryAttemptsCount - retryCount} attempt left."); Trace.Warning($"Back off {backOff.TotalSeconds} seconds before next retry. {maxRetryAttemptsCount - retryCount} attempt left.");

View File

@@ -1,8 +1,8 @@
using GitHub.Runner.Sdk;
using System; using System;
using System.Diagnostics; using System.Diagnostics;
using System.Globalization; using System.Globalization;
using System.IO; using System.IO;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Common namespace GitHub.Runner.Common
{ {
@@ -24,17 +24,10 @@ namespace GitHub.Runner.Common
return; return;
} }
if (!string.IsNullOrEmpty(message))
{
var messageLines = message.Split(Environment.NewLine);
foreach (var messageLine in messageLines)
{
WriteHeader(source, eventType, id); WriteHeader(source, eventType, id);
WriteLine(messageLine); WriteLine(message);
WriteFooter(eventCache); WriteFooter(eventCache);
} }
}
}
internal bool IsEnabled(TraceOptions opts) internal bool IsEnabled(TraceOptions opts)
{ {
@@ -94,3 +87,4 @@ namespace GitHub.Runner.Common
} }
} }
} }

View File

@@ -1,8 +0,0 @@
// Represents absence of value.
namespace GitHub.Runner.Common
{
public readonly struct Unit
{
public static readonly Unit Value = default;
}
}

View File

@@ -1,14 +0,0 @@
namespace GitHub.Runner.Common.Util
{
using System;
using GitHub.DistributedTask.WebApi;
public static class MessageUtil
{
public static bool IsRunServiceJob(string messageType)
{
return string.Equals(messageType, JobRequestMessageTypes.RunnerJobRequest, StringComparison.OrdinalIgnoreCase);
}
}
}

View File

@@ -56,8 +56,7 @@ namespace GitHub.Runner.Listener
new string[] new string[]
{ {
Constants.Runner.CommandLine.Args.Token, Constants.Runner.CommandLine.Args.Token,
Constants.Runner.CommandLine.Args.PAT, Constants.Runner.CommandLine.Args.PAT
Constants.Runner.CommandLine.Flags.Local
}, },
// Valid run flags and args // Valid run flags and args
[Constants.Runner.CommandLine.Commands.Run] = [Constants.Runner.CommandLine.Commands.Run] =
@@ -87,7 +86,6 @@ namespace GitHub.Runner.Listener
public bool Help => TestFlag(Constants.Runner.CommandLine.Flags.Help); public bool Help => TestFlag(Constants.Runner.CommandLine.Flags.Help);
public bool Unattended => TestFlag(Constants.Runner.CommandLine.Flags.Unattended); public bool Unattended => TestFlag(Constants.Runner.CommandLine.Flags.Unattended);
public bool Version => TestFlag(Constants.Runner.CommandLine.Flags.Version); public bool Version => TestFlag(Constants.Runner.CommandLine.Flags.Version);
public bool RemoveLocalConfig => TestFlag(Constants.Runner.CommandLine.Flags.Local);
// Keep this around since customers still relies on it // Keep this around since customers still relies on it
public bool RunOnce => TestFlag(Constants.Runner.CommandLine.Flags.Once); public bool RunOnce => TestFlag(Constants.Runner.CommandLine.Flags.Once);

View File

@@ -1,3 +1,10 @@
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using GitHub.Services.Common;
using GitHub.Services.Common.Internal;
using GitHub.Services.OAuth;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
@@ -7,13 +14,6 @@ using System.Runtime.InteropServices;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text; using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk;
using GitHub.Services.Common;
using GitHub.Services.Common.Internal;
using GitHub.Services.OAuth;
namespace GitHub.Runner.Listener.Configuration namespace GitHub.Runner.Listener.Configuration
{ {
@@ -636,7 +636,7 @@ namespace GitHub.Runner.Listener.Configuration
} }
int retryCount = 0; int retryCount = 0;
while (retryCount < 3) while(retryCount < 3)
{ {
using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler)) using (var httpClient = new HttpClient(httpClientHandler))
@@ -652,23 +652,22 @@ namespace GitHub.Runner.Listener.Configuration
{ {
var response = await httpClient.PostAsync(githubApiUrl, new StringContent(string.Empty)); var response = await httpClient.PostAsync(githubApiUrl, new StringContent(string.Empty));
responseStatus = response.StatusCode; responseStatus = response.StatusCode;
var githubRequestId = GetGitHubRequestId(response.Headers);
if (response.IsSuccessStatusCode) if (response.IsSuccessStatusCode)
{ {
Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}' ({githubRequestId})"); Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var jsonResponse = await response.Content.ReadAsStringAsync(); var jsonResponse = await response.Content.ReadAsStringAsync();
return StringUtil.ConvertFromJson<GitHubRunnerRegisterToken>(jsonResponse); return StringUtil.ConvertFromJson<GitHubRunnerRegisterToken>(jsonResponse);
} }
else else
{ {
_term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}' (Request Id: {githubRequestId})"); _term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var errorResponse = await response.Content.ReadAsStringAsync(); var errorResponse = await response.Content.ReadAsStringAsync();
_term.WriteError(errorResponse); _term.WriteError(errorResponse);
response.EnsureSuccessStatusCode(); response.EnsureSuccessStatusCode();
} }
} }
catch (Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound) catch(Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound)
{ {
retryCount++; retryCount++;
Trace.Error($"Failed to get JIT runner token -- Atempt: {retryCount}"); Trace.Error($"Failed to get JIT runner token -- Atempt: {retryCount}");
@@ -715,23 +714,22 @@ namespace GitHub.Runner.Listener.Configuration
{ {
var response = await httpClient.PostAsync(githubApiUrl, new StringContent(StringUtil.ConvertToJson(bodyObject), null, "application/json")); var response = await httpClient.PostAsync(githubApiUrl, new StringContent(StringUtil.ConvertToJson(bodyObject), null, "application/json"));
responseStatus = response.StatusCode; responseStatus = response.StatusCode;
var githubRequestId = GetGitHubRequestId(response.Headers);
if (response.IsSuccessStatusCode) if(response.IsSuccessStatusCode)
{ {
Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}' ({githubRequestId})"); Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var jsonResponse = await response.Content.ReadAsStringAsync(); var jsonResponse = await response.Content.ReadAsStringAsync();
return StringUtil.ConvertFromJson<GitHubAuthResult>(jsonResponse); return StringUtil.ConvertFromJson<GitHubAuthResult>(jsonResponse);
} }
else else
{ {
_term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}' (Request Id: {githubRequestId})"); _term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var errorResponse = await response.Content.ReadAsStringAsync(); var errorResponse = await response.Content.ReadAsStringAsync();
_term.WriteError(errorResponse); _term.WriteError(errorResponse);
response.EnsureSuccessStatusCode(); response.EnsureSuccessStatusCode();
} }
} }
catch (Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound) catch(Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound)
{ {
retryCount++; retryCount++;
Trace.Error($"Failed to get tenant credentials -- Atempt: {retryCount}"); Trace.Error($"Failed to get tenant credentials -- Atempt: {retryCount}");
@@ -744,14 +742,5 @@ namespace GitHub.Runner.Listener.Configuration
} }
return null; return null;
} }
private string GetGitHubRequestId(HttpResponseHeaders headers)
{
if (headers.TryGetValues("x-github-request-id", out var headerValues))
{
return headerValues.FirstOrDefault();
}
return string.Empty;
}
} }
} }

View File

@@ -1,3 +0,0 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("Test")]

View File

@@ -7,7 +7,6 @@ using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.DistributedTask.Pipelines;
using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common; using GitHub.Runner.Common;
using GitHub.Runner.Common.Util; using GitHub.Runner.Common.Util;
@@ -59,8 +58,6 @@ namespace GitHub.Runner.Listener
public event EventHandler<JobStatusEventArgs> JobStatus; public event EventHandler<JobStatusEventArgs> JobStatus;
private bool _isRunServiceJob;
public override void Initialize(IHostContext hostContext) public override void Initialize(IHostContext hostContext)
{ {
base.Initialize(hostContext); base.Initialize(hostContext);
@@ -89,8 +86,6 @@ namespace GitHub.Runner.Listener
{ {
Trace.Info($"Job request {jobRequestMessage.RequestId} for plan {jobRequestMessage.Plan.PlanId} job {jobRequestMessage.JobId} received."); Trace.Info($"Job request {jobRequestMessage.RequestId} for plan {jobRequestMessage.Plan.PlanId} job {jobRequestMessage.JobId} received.");
_isRunServiceJob = MessageUtil.IsRunServiceJob(jobRequestMessage.MessageType);
WorkerDispatcher currentDispatch = null; WorkerDispatcher currentDispatch = null;
if (_jobDispatchedQueue.Count > 0) if (_jobDispatchedQueue.Count > 0)
{ {
@@ -244,13 +239,6 @@ namespace GitHub.Runner.Listener
return; return;
} }
if (this._isRunServiceJob)
{
Trace.Error($"We are not yet checking the state of jobrequest {jobDispatch.JobId} status. Cancel running worker right away.");
jobDispatch.WorkerCancellationTokenSource.Cancel();
return;
}
// based on the current design, server will only send one job for a given runner at a time. // based on the current design, server will only send one job for a given runner at a time.
// if the runner received a new job request while a previous job request is still running, this typically indicates two situations // if the runner received a new job request while a previous job request is still running, this typically indicates two situations
// 1. a runner bug caused a server and runner mismatch on the state of the job request, e.g. the runner didn't renew the jobrequest // 1. a runner bug caused a server and runner mismatch on the state of the job request, e.g. the runner didn't renew the jobrequest
@@ -379,11 +367,9 @@ namespace GitHub.Runner.Listener
long requestId = message.RequestId; long requestId = message.RequestId;
Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat
var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
// start renew job request // start renew job request
Trace.Info($"Start renew job request {requestId} for job {message.JobId}."); Trace.Info($"Start renew job request {requestId} for job {message.JobId}.");
Task renewJobRequest = RenewJobRequestAsync(message, systemConnection, _poolId, requestId, lockToken, orchestrationId, firstJobRequestRenewed, lockRenewalTokenSource.Token); Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, orchestrationId, firstJobRequestRenewed, lockRenewalTokenSource.Token);
// wait till first renew succeed or job request is cancelled // wait till first renew succeed or job request is cancelled
// not even start worker if the first renew fail // not even start worker if the first renew fail
@@ -522,10 +508,11 @@ namespace GitHub.Runner.Listener
// we get first jobrequest renew succeed and start the worker process with the job message. // we get first jobrequest renew succeed and start the worker process with the job message.
// send notification to machine provisioner. // send notification to machine provisioner.
var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
var accessToken = systemConnection?.Authorization?.Parameters["AccessToken"]; var accessToken = systemConnection?.Authorization?.Parameters["AccessToken"];
notification.JobStarted(message.JobId, accessToken, systemConnection.Url); notification.JobStarted(message.JobId, accessToken, systemConnection.Url);
HostContext.WritePerfCounter($"SentJobToWorker_{requestId.ToString()}"); HostContext.WritePerfCounter($"SentJobToWorker_{requestId}");
try try
{ {
@@ -544,8 +531,11 @@ namespace GitHub.Runner.Listener
detailInfo = string.Join(Environment.NewLine, workerOutput); detailInfo = string.Join(Environment.NewLine, workerOutput);
Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result."); Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result.");
var jobServer = HostContext.GetService<IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
VssConnection jobConnection = VssUtil.CreateConnection(systemConnection.Url, jobServerCredential);
await jobServer.ConnectAsync(jobConnection);
var jobServer = await InitializeJobServerAsync(systemConnection);
await LogWorkerProcessUnhandledException(jobServer, message, detailInfo); await LogWorkerProcessUnhandledException(jobServer, message, detailInfo);
// Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space. // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space.
@@ -685,128 +675,9 @@ namespace GitHub.Runner.Listener
} }
} }
internal async Task RenewJobRequestAsync(Pipelines.AgentJobRequestMessage message, ServiceEndpoint systemConnection, int poolId, long requestId, Guid lockToken, string orchestrationId, TaskCompletionSource<int> firstJobRequestRenewed, CancellationToken token) public async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockToken, string orchestrationId, TaskCompletionSource<int> firstJobRequestRenewed, CancellationToken token)
{
if (this._isRunServiceJob)
{
var runServer = await GetRunServerAsync(systemConnection);
await RenewJobRequestAsync(runServer, message.Plan.PlanId, message.JobId, firstJobRequestRenewed, token);
}
else
{ {
var runnerServer = HostContext.GetService<IRunnerServer>(); var runnerServer = HostContext.GetService<IRunnerServer>();
await RenewJobRequestAsync(runnerServer, poolId, requestId, lockToken, orchestrationId, firstJobRequestRenewed, token);
}
}
private async Task RenewJobRequestAsync(IRunServer runServer, Guid planId, Guid jobId, TaskCompletionSource<int> firstJobRequestRenewed, CancellationToken token)
{
TaskAgentJobRequest request = null;
int firstRenewRetryLimit = 5;
int encounteringError = 0;
// renew lock during job running.
// stop renew only if cancellation token for lock renew task been signal or exception still happen after retry.
while (!token.IsCancellationRequested)
{
try
{
var renewResponse = await runServer.RenewJobAsync(planId, jobId, token);
Trace.Info($"Successfully renew job {jobId}, job is valid till {renewResponse.LockedUntil}");
if (!firstJobRequestRenewed.Task.IsCompleted)
{
// fire first renew succeed event.
firstJobRequestRenewed.TrySetResult(0);
}
if (encounteringError > 0)
{
encounteringError = 0;
HostContext.WritePerfCounter("JobRenewRecovered");
}
// renew again after 60 sec delay
await HostContext.Delay(TimeSpan.FromSeconds(60), token);
}
catch (TaskOrchestrationJobNotFoundException)
{
// no need for retry. the job is not valid anymore.
Trace.Info($"TaskAgentJobNotFoundException received when renew job {jobId}, job is no longer valid, stop renew job request.");
return;
}
catch (OperationCanceledException) when (token.IsCancellationRequested)
{
// OperationCanceledException may caused by http timeout or _lockRenewalTokenSource.Cance();
// Stop renew only on cancellation token fired.
Trace.Info($"job renew has been cancelled, stop renew job {jobId}.");
return;
}
catch (Exception ex)
{
Trace.Error($"Catch exception during renew runner job {jobId}.");
Trace.Error(ex);
encounteringError++;
// retry
TimeSpan remainingTime = TimeSpan.Zero;
if (!firstJobRequestRenewed.Task.IsCompleted)
{
// retry 5 times every 10 sec for the first renew
if (firstRenewRetryLimit-- > 0)
{
remainingTime = TimeSpan.FromSeconds(10);
}
}
else
{
// retry till reach lockeduntil + 5 mins extra buffer.
remainingTime = request.LockedUntil.Value + TimeSpan.FromMinutes(5) - DateTime.UtcNow;
}
if (remainingTime > TimeSpan.Zero)
{
TimeSpan delayTime;
if (!firstJobRequestRenewed.Task.IsCompleted)
{
Trace.Info($"Retrying lock renewal for job {jobId}. The first job renew request has failed.");
delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10));
}
else
{
Trace.Info($"Retrying lock renewal for job {jobId}. Job is valid until {request.LockedUntil.Value}.");
if (encounteringError > 5)
{
delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30));
}
else
{
delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15));
}
}
try
{
// back-off before next retry.
await HostContext.Delay(delayTime, token);
}
catch (OperationCanceledException) when (token.IsCancellationRequested)
{
Trace.Info($"job renew has been cancelled, stop renew job {jobId}.");
}
}
else
{
Trace.Info($"Lock renewal has run out of retry, stop renew lock for job {jobId}.");
HostContext.WritePerfCounter("JobRenewReachLimit");
return;
}
}
}
}
private async Task RenewJobRequestAsync(IRunnerServer runnerServer, int poolId, long requestId, Guid lockToken, string orchestrationId, TaskCompletionSource<int> firstJobRequestRenewed, CancellationToken token)
{
TaskAgentJobRequest request = null; TaskAgentJobRequest request = null;
int firstRenewRetryLimit = 5; int firstRenewRetryLimit = 5;
int encounteringError = 0; int encounteringError = 0;
@@ -969,10 +840,12 @@ namespace GitHub.Runner.Listener
var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection)); var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection));
ArgUtil.NotNull(systemConnection, nameof(systemConnection)); ArgUtil.NotNull(systemConnection, nameof(systemConnection));
var server = await InitializeJobServerAsync(systemConnection); var jobServer = HostContext.GetService<IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
VssConnection jobConnection = VssUtil.CreateConnection(systemConnection.Url, jobServerCredential);
await jobServer.ConnectAsync(jobConnection);
if (server is IJobServer jobServer)
{
var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None); var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None);
var updatedRecords = new List<TimelineRecord>(); var updatedRecords = new List<TimelineRecord>();
@@ -1053,11 +926,6 @@ namespace GitHub.Runner.Listener
await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, updatedRecords, CancellationToken.None); await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, updatedRecords, CancellationToken.None);
} }
} }
else
{
Trace.Info("Job server does not support log upload yet.");
}
}
catch (Exception ex) catch (Exception ex)
{ {
// Ignore any error during log upload since it's best effort // Ignore any error during log upload since it's best effort
@@ -1075,12 +943,6 @@ namespace GitHub.Runner.Listener
return; return;
} }
if (this._isRunServiceJob)
{
Trace.Verbose($"Skip FinishAgentRequest call from Listener because MessageType is {message.MessageType}");
return;
}
var runnerServer = HostContext.GetService<IRunnerServer>(); var runnerServer = HostContext.GetService<IRunnerServer>();
int completeJobRequestRetryLimit = 5; int completeJobRequestRetryLimit = 5;
List<Exception> exceptions = new(); List<Exception> exceptions = new();
@@ -1117,9 +979,7 @@ namespace GitHub.Runner.Listener
} }
// log an error issue to job level timeline record // log an error issue to job level timeline record
private async Task LogWorkerProcessUnhandledException(IRunnerService server, Pipelines.AgentJobRequestMessage message, string errorMessage) private async Task LogWorkerProcessUnhandledException(IJobServer jobServer, Pipelines.AgentJobRequestMessage message, string errorMessage)
{
if (server is IJobServer jobServer)
{ {
try try
{ {
@@ -1154,7 +1014,7 @@ namespace GitHub.Runner.Listener
} }
var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = errorMessage }; var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = errorMessage };
unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash; unhandledExceptionIssue[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash;
jobRecord.ErrorCount++; jobRecord.ErrorCount++;
jobRecord.Issues.Add(unhandledExceptionIssue); jobRecord.Issues.Add(unhandledExceptionIssue);
await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None); await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None);
@@ -1165,17 +1025,9 @@ namespace GitHub.Runner.Listener
Trace.Error(ex); Trace.Error(ex);
} }
} }
else
{
Trace.Info("Job server does not support handling unhandled exception yet, error message: {0}", errorMessage);
return;
}
}
// raise job completed event to fail the job. // raise job completed event to fail the job.
private async Task ForceFailJob(IRunnerService server, Pipelines.AgentJobRequestMessage message) private async Task ForceFailJob(IJobServer jobServer, Pipelines.AgentJobRequestMessage message)
{
if (server is IJobServer jobServer)
{ {
try try
{ {
@@ -1188,47 +1040,6 @@ namespace GitHub.Runner.Listener
Trace.Error(ex); Trace.Error(ex);
} }
} }
else if (server is IRunServer runServer)
{
try
{
await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, CancellationToken.None);
}
catch (Exception ex)
{
Trace.Error("Fail to raise job completion back to service.");
Trace.Error(ex);
}
}
else
{
throw new NotSupportedException($"Server type {server.GetType().FullName} is not supported.");
}
}
private async Task<IRunnerService> InitializeJobServerAsync(ServiceEndpoint systemConnection)
{
if (this._isRunServiceJob)
{
return await GetRunServerAsync(systemConnection);
}
else
{
var jobServer = HostContext.GetService<IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
VssConnection jobConnection = VssUtil.CreateConnection(systemConnection.Url, jobServerCredential);
await jobServer.ConnectAsync(jobConnection);
return jobServer;
}
}
private async Task<IRunServer> GetRunServerAsync(ServiceEndpoint systemConnection)
{
var runServer = HostContext.GetService<IRunServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
await runServer.ConnectAsync(systemConnection.Url, jobServerCredential);
return runServer;
}
private class WorkerDispatcher : IDisposable private class WorkerDispatcher : IDisposable
{ {

View File

@@ -9,7 +9,6 @@ using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common; using GitHub.Runner.Common;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Listener.Check; using GitHub.Runner.Listener.Check;
using GitHub.Runner.Listener.Configuration; using GitHub.Runner.Listener.Configuration;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;
@@ -136,12 +135,6 @@ namespace GitHub.Runner.Listener
// remove config files, remove service, and exit // remove config files, remove service, and exit
if (command.Remove) if (command.Remove)
{ {
// only remove local config files and exit
if (command.RemoveLocalConfig)
{
configManager.DeleteLocalRunnerConfig();
return Constants.Runner.ReturnCode.Success;
}
try try
{ {
await configManager.UnconfigureAsync(command); await configManager.UnconfigureAsync(command);
@@ -503,7 +496,7 @@ namespace GitHub.Runner.Listener
} }
} }
// Broker flow // Broker flow
else if (MessageUtil.IsRunServiceJob(message.MessageType)) else if (string.Equals(message.MessageType, JobRequestMessageTypes.RunnerJobRequest, StringComparison.OrdinalIgnoreCase))
{ {
if (autoUpdateInProgress || runOnceJobReceived) if (autoUpdateInProgress || runOnceJobReceived)
{ {
@@ -654,7 +647,6 @@ Config Options:
--name string Name of the runner to configure (default {Environment.MachineName ?? "myrunner"}) --name string Name of the runner to configure (default {Environment.MachineName ?? "myrunner"})
--runnergroup string Name of the runner group to add this runner to (defaults to the default runner group) --runnergroup string Name of the runner group to add this runner to (defaults to the default runner group)
--labels string Extra labels in addition to the default: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}' --labels string Extra labels in addition to the default: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}'
--local Removes the runner config files from your local machine. Used as an option to the remove command
--work string Relative runner work directory (default {Constants.Path.WorkDirectory}) --work string Relative runner work directory (default {Constants.Path.WorkDirectory})
--replace Replace any existing runner with the same name (default false) --replace Replace any existing runner with the same name (default false)
--pat GitHub personal access token with repo scope. Used for checking network connectivity when executing `.{separator}run.{ext} --check` --pat GitHub personal access token with repo scope. Used for checking network connectivity when executing `.{separator}run.{ext} --check`

View File

@@ -73,7 +73,7 @@ namespace GitHub.Runner.Sdk
{ {
var headerValues = new List<ProductInfoHeaderValue>(); var headerValues = new List<ProductInfoHeaderValue>();
headerValues.Add(new ProductInfoHeaderValue($"GitHubActionsRunner-Plugin", BuildConstants.RunnerPackage.Version)); headerValues.Add(new ProductInfoHeaderValue($"GitHubActionsRunner-Plugin", BuildConstants.RunnerPackage.Version));
headerValues.Add(new ProductInfoHeaderValue($"({StringUtil.SanitizeUserAgentHeader(RuntimeInformation.OSDescription)})")); headerValues.Add(new ProductInfoHeaderValue($"({RuntimeInformation.OSDescription.Trim()})"));
if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0) if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0)
{ {

View File

@@ -264,17 +264,7 @@ namespace GitHub.Runner.Sdk
{ {
foreach (KeyValuePair<string, string> kvp in environment) foreach (KeyValuePair<string, string> kvp in environment)
{ {
#if OS_WINDOWS
string tempKey = String.IsNullOrWhiteSpace(kvp.Key) ? kvp.Key : kvp.Key.Split('\0')[0];
string tempValue = String.IsNullOrWhiteSpace(kvp.Value) ? kvp.Value : kvp.Value.Split('\0')[0];
if(!String.IsNullOrWhiteSpace(tempKey))
{
_proc.StartInfo.Environment[tempKey] = tempValue;
}
#else
_proc.StartInfo.Environment[kvp.Key] = kvp.Value; _proc.StartInfo.Environment[kvp.Key] = kvp.Value;
#endif
} }
} }

View File

@@ -164,6 +164,7 @@ namespace GitHub.Runner.Sdk
{ {
continue; continue;
} }
_noProxyList.Add(noProxyInfo); _noProxyList.Add(noProxyInfo);
} }
} }
@@ -206,11 +207,6 @@ namespace GitHub.Runner.Sdk
{ {
foreach (var noProxy in _noProxyList) foreach (var noProxy in _noProxyList)
{ {
// bypass on wildcard no_proxy
if (string.Equals(noProxy.Host, "*", StringComparison.OrdinalIgnoreCase))
{
return true;
}
var matchHost = false; var matchHost = false;
var matchPort = false; var matchPort = false;

View File

@@ -123,12 +123,5 @@ namespace GitHub.Runner.Sdk
{ {
return value?.Substring(0, Math.Min(value.Length, count)); return value?.Substring(0, Math.Min(value.Length, count));
} }
// Fixes format violations e.g. https://github.com/actions/runner/issues/2165
public static string SanitizeUserAgentHeader(string header)
{
return header.Replace("(", "[").Replace(")", "]").Trim();
}
} }
} }

View File

@@ -6,16 +6,9 @@ namespace GitHub.Runner.Sdk
{ {
public static bool IsHostedServer(UriBuilder gitHubUrl) public static bool IsHostedServer(UriBuilder gitHubUrl)
{ {
if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_FORCE_GHES"))) return string.Equals(gitHubUrl.Host, "github.com", StringComparison.OrdinalIgnoreCase) ||
{
return false;
}
return
string.Equals(gitHubUrl.Host, "github.com", StringComparison.OrdinalIgnoreCase) ||
string.Equals(gitHubUrl.Host, "www.github.com", StringComparison.OrdinalIgnoreCase) || string.Equals(gitHubUrl.Host, "www.github.com", StringComparison.OrdinalIgnoreCase) ||
string.Equals(gitHubUrl.Host, "github.localhost", StringComparison.OrdinalIgnoreCase) || string.Equals(gitHubUrl.Host, "github.localhost", StringComparison.OrdinalIgnoreCase);
gitHubUrl.Host.EndsWith(".ghe.com", StringComparison.OrdinalIgnoreCase);
} }
public static Uri GetCredentialEmbeddedUrl(Uri baseUrl, string username, string password) public static Uri GetCredentialEmbeddedUrl(Uri baseUrl, string username, string password)

View File

@@ -19,7 +19,7 @@ namespace GitHub.Runner.Sdk
{ {
var headerValues = new List<ProductInfoHeaderValue>(); var headerValues = new List<ProductInfoHeaderValue>();
headerValues.AddRange(additionalUserAgents); headerValues.AddRange(additionalUserAgents);
headerValues.Add(new ProductInfoHeaderValue($"({StringUtil.SanitizeUserAgentHeader(RuntimeInformation.OSDescription)})")); headerValues.Add(new ProductInfoHeaderValue($"({RuntimeInformation.OSDescription.Trim()})"));
if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0) if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0)
{ {
@@ -116,7 +116,7 @@ namespace GitHub.Runner.Sdk
// settings are applied to an HttpRequestMessage. // settings are applied to an HttpRequestMessage.
settings.AcceptLanguages.Remove(CultureInfo.InvariantCulture); settings.AcceptLanguages.Remove(CultureInfo.InvariantCulture);
RawConnection connection = new(serverUri, new RawHttpMessageHandler(credentials.Federated, settings), additionalDelegatingHandler); RawConnection connection = new(serverUri, new RawHttpMessageHandler(credentials.ToOAuthCredentials(), settings), additionalDelegatingHandler);
return connection; return connection;
} }

View File

@@ -270,12 +270,9 @@ namespace GitHub.Runner.Worker
if (string.Equals(blocked, envName, StringComparison.OrdinalIgnoreCase)) if (string.Equals(blocked, envName, StringComparison.OrdinalIgnoreCase))
{ {
// Log Telemetry and let user know they shouldn't do this // Log Telemetry and let user know they shouldn't do this
var issue = new Issue() var message = $"Can't update {blocked} environment variable using ::set-env:: command.";
{ var metadata = new IssueMetadata(Constants.Runner.InternalTelemetryIssueDataKey, $"{Constants.Runner.UnsupportedCommand}_{envName}");
Type = IssueType.Error, var issue = context.CreateIssue(IssueType.Error, message, metadata, true);
Message = $"Can't update {blocked} environment variable using ::set-env:: command."
};
issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = $"{Constants.Runner.UnsupportedCommand}_{envName}";
context.AddIssue(issue); context.AddIssue(issue);
return; return;
@@ -309,12 +306,9 @@ namespace GitHub.Runner.Worker
{ {
if (context.Global.Variables.GetBoolean("DistributedTask.DeprecateStepOutputCommands") ?? false) if (context.Global.Variables.GetBoolean("DistributedTask.DeprecateStepOutputCommands") ?? false)
{ {
var issue = new Issue() var message = string.Format(Constants.Runner.UnsupportedCommandMessage, this.Command);
{ var metadata = new IssueMetadata(Constants.Runner.InternalTelemetryIssueDataKey, Constants.Runner.UnsupportedCommand);
Type = IssueType.Warning, var issue = context.CreateIssue(IssueType.Warning, message, metadata, true);
Message = String.Format(Constants.Runner.UnsupportedCommandMessage, this.Command)
};
issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.UnsupportedCommand;
context.AddIssue(issue); context.AddIssue(issue);
} }
@@ -344,12 +338,9 @@ namespace GitHub.Runner.Worker
{ {
if (context.Global.Variables.GetBoolean("DistributedTask.DeprecateStepOutputCommands") ?? false) if (context.Global.Variables.GetBoolean("DistributedTask.DeprecateStepOutputCommands") ?? false)
{ {
var issue = new Issue() var message = string.Format(Constants.Runner.UnsupportedCommandMessage, this.Command);
{ var metadata = new IssueMetadata(Constants.Runner.InternalTelemetryIssueDataKey, Constants.Runner.UnsupportedCommand);
Type = IssueType.Warning, var issue = context.CreateIssue(IssueType.Warning, message, metadata, true);
Message = String.Format(Constants.Runner.UnsupportedCommandMessage, this.Command)
};
issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.UnsupportedCommand;
context.AddIssue(issue); context.AddIssue(issue);
} }
@@ -618,16 +609,11 @@ namespace GitHub.Runner.Worker
context.Debug("Enhanced Annotations not enabled on the server. The 'title', 'end_line', and 'end_column' fields are unsupported."); context.Debug("Enhanced Annotations not enabled on the server. The 'title', 'end_line', and 'end_column' fields are unsupported.");
} }
Issue issue = new() var issueCategory = "General";
{
Category = "General",
Type = this.Type,
Message = command.Data
};
if (!string.IsNullOrEmpty(file)) if (!string.IsNullOrEmpty(file))
{ {
issue.Category = "Code"; issueCategory = "Code";
if (container != null) if (container != null)
{ {
@@ -658,14 +644,13 @@ namespace GitHub.Runner.Worker
} }
} }
foreach (var property in command.Properties) string keyToExclude = Constants.Runner.InternalTelemetryIssueDataKey;
{ var filteredDictionaryEntries = command.Properties
if (!string.Equals(property.Key, Constants.Runner.InternalTelemetryIssueDataKey, StringComparison.OrdinalIgnoreCase)) .Where(kvp => !string.Equals(kvp.Key, keyToExclude, StringComparison.OrdinalIgnoreCase))
{ .ToList();
issue.Data[property.Key] = property.Value;
}
}
var metadata = new IssueMetadata(issueCategory, false, null, filteredDictionaryEntries);
var issue = context.CreateIssue(this.Type, command.Data, metadata, true);
context.AddIssue(issue); context.AddIssue(issue);
} }

View File

@@ -1,7 +1,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.IO.Compression; using System.IO.Compression; // required for OS_WINDOWS
using System.Linq; using System.Linq;
using System.Net; using System.Net;
using System.Net.Http; using System.Net.Http;

View File

@@ -25,6 +25,7 @@ namespace GitHub.Runner.Worker
public interface IActionRunner : IStep, IRunnerService public interface IActionRunner : IStep, IRunnerService
{ {
ActionRunStage Stage { get; set; } ActionRunStage Stage { get; set; }
bool TryEvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context);
Pipelines.ActionStep Action { get; set; } Pipelines.ActionStep Action { get; set; }
} }
@@ -181,7 +182,7 @@ namespace GitHub.Runner.Worker
else else
{ {
var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator(); var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator();
inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, ExecutionContext.ExpressionValues, ExecutionContext.ExpressionFunctions, ExecutionContext.ToExpressionState()); inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, ExecutionContext.ExpressionValues, ExecutionContext.ExpressionFunctions);
} }
var userInputs = new HashSet<string>(StringComparer.OrdinalIgnoreCase); var userInputs = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
@@ -284,67 +285,25 @@ namespace GitHub.Runner.Worker
} }
/// <summary> public bool TryEvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context)
/// Attempts to update the DisplayName.
/// As the "Try..." name implies, this method should never throw an exception.
/// Returns true if the DisplayName is already present or it was successfully updated.
/// </summary>
public bool TryUpdateDisplayName(out bool updated)
{
updated = false;
// REVIEW: This try/catch can be removed if some future implementation of EvaluateDisplayName and UpdateTimelineRecordDisplayName
// can make reasonable guarantees that they won't throw an exception.
try
{
// This attempt is only worthwhile at the "Main" stage.
// When the job starts, there's an initial attempt to evaluate the DisplayName. (see JobExtension::InitializeJob)
// During the "Pre" stage, we expect that no contexts will have changed since the initial evaluation.
// "Main" stage is handled here.
// During the "Post" stage, it no longer matters.
if (this.Stage == ActionRunStage.Main && EvaluateDisplayName(this.ExecutionContext.ExpressionValues, this.ExecutionContext, out updated))
{
if (updated)
{
this.ExecutionContext.UpdateTimelineRecordDisplayName(this.DisplayName);
}
}
}
catch (Exception ex)
{
Trace.Warning("Caught exception while attempting to evaulate/update the step's DisplayName. Exception Details: {0}", ex);
}
// For consistency with other implementations of TryUpdateDisplayName we use !string.IsNullOrEmpty below,
// but note that (at the time of this writing) ActionRunner::DisplayName::get always returns a non-empty string due to its fallback logic.
// In other words, the net effect is that this particular implementation of TryUpdateDisplayName will always return true.
return !string.IsNullOrEmpty(this.DisplayName);
}
/// <summary>
/// Attempts to evaluate the DisplayName of this IActionRunner.
/// Returns true if the DisplayName is already present or it was successfully evaluated.
/// </summary>
public bool EvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context, out bool updated)
{ {
ArgUtil.NotNull(context, nameof(context)); ArgUtil.NotNull(context, nameof(context));
ArgUtil.NotNull(Action, nameof(Action)); ArgUtil.NotNull(Action, nameof(Action));
updated = false; // If we have already expanded the display name, there is no need to expand it again
// If we have already expanded the display name, don't bother attempting [re-]expansion. // TODO: Remove the ShouldEvaluateDisplayName check and field post m158 deploy, we should do it by default once the server is updated
if (_didFullyEvaluateDisplayName || !string.IsNullOrEmpty(Action.DisplayName)) if (_didFullyEvaluateDisplayName || !string.IsNullOrEmpty(Action.DisplayName))
{ {
return true; return false;
} }
_displayName = GenerateDisplayName(Action, contextData, context, out bool didFullyEvaluate); bool didFullyEvaluate;
_displayName = GenerateDisplayName(Action, contextData, context, out didFullyEvaluate);
// If we evaluated, fully mask any secrets // If we evaluated fully mask any secrets
if (didFullyEvaluate) if (didFullyEvaluate)
{ {
_displayName = HostContext.SecretMasker.MaskSecrets(_displayName); _displayName = HostContext.SecretMasker.MaskSecrets(_displayName);
updated = true;
} }
context.Debug($"Set step '{Action.Name}' display name to: '{_displayName}'"); context.Debug($"Set step '{Action.Name}' display name to: '{_displayName}'");
_didFullyEvaluateDisplayName = didFullyEvaluate; _didFullyEvaluateDisplayName = didFullyEvaluate;
@@ -355,7 +314,7 @@ namespace GitHub.Runner.Worker
{ {
DictionaryContextData expressionValues = ExecutionContext.GetExpressionValues(stepHost); DictionaryContextData expressionValues = ExecutionContext.GetExpressionValues(stepHost);
var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator(); var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator();
var inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, expressionValues, ExecutionContext.ExpressionFunctions, ExecutionContext.ToExpressionState()); var inputs = templateEvaluator.EvaluateStepInputs(Action.Inputs, expressionValues, ExecutionContext.ExpressionFunctions);
return inputs; return inputs;
} }

View File

@@ -33,15 +33,9 @@ namespace GitHub.Runner.Worker
public override void Initialize(IHostContext hostContext) public override void Initialize(IHostContext hostContext)
{ {
base.Initialize(hostContext); base.Initialize(hostContext);
if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(Constants.Hooks.ContainerHooksPath)))
{
_dockerManager = HostContext.GetService<IDockerCommandManager>(); _dockerManager = HostContext.GetService<IDockerCommandManager>();
}
else
{
_containerHookManager = HostContext.GetService<IContainerHookManager>(); _containerHookManager = HostContext.GetService<IContainerHookManager>();
} }
}
public async Task StartContainersAsync(IExecutionContext executionContext, object data) public async Task StartContainersAsync(IExecutionContext executionContext, object data)
{ {

View File

@@ -5,7 +5,6 @@ using System.IO.Compression;
using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.WebApi;
using System.Linq; using System.Linq;
using System.Globalization; using System.Globalization;
using System.Threading.Tasks;
using Pipelines = GitHub.DistributedTask.Pipelines; using Pipelines = GitHub.DistributedTask.Pipelines;
using GitHub.Runner.Common; using GitHub.Runner.Common;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;

View File

@@ -1,4 +1,4 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Globalization; using System.Globalization;
using System.IO; using System.IO;
@@ -6,7 +6,6 @@ using System.Linq;
using System.Text; using System.Text;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.Actions.RunService.WebApi;
using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.Expressions2;
using GitHub.DistributedTask.ObjectTemplating.Tokens; using GitHub.DistributedTask.ObjectTemplating.Tokens;
using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.Pipelines.ContextData;
@@ -17,6 +16,7 @@ using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;
using GitHub.Runner.Worker.Container; using GitHub.Runner.Worker.Container;
using GitHub.Runner.Worker.Handlers; using GitHub.Runner.Worker.Handlers;
using GitHub.Services.Common;
using Newtonsoft.Json; using Newtonsoft.Json;
using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating; using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating;
using Pipelines = GitHub.DistributedTask.Pipelines; using Pipelines = GitHub.DistributedTask.Pipelines;
@@ -81,7 +81,6 @@ namespace GitHub.Runner.Worker
// logging // logging
long Write(string tag, string message); long Write(string tag, string message);
void QueueAttachFile(string type, string name, string filePath); void QueueAttachFile(string type, string name, string filePath);
void QueueSummaryFile(string name, string filePath, Guid stepRecordId);
// timeline record update methods // timeline record update methods
void Start(string currentOperation = null); void Start(string currentOperation = null);
@@ -92,7 +91,8 @@ namespace GitHub.Runner.Worker
void SetGitHubContext(string name, string value); void SetGitHubContext(string name, string value);
void SetOutput(string name, string value, out string reference); void SetOutput(string name, string value, out string reference);
void SetTimeout(TimeSpan? timeout); void SetTimeout(TimeSpan? timeout);
void AddIssue(Issue issue, string message = null); IReadOnlyIssue CreateIssue(IssueType issueType, string rawMessage, IssueMetadata metadata, bool writeToLog);
void AddIssue(IReadOnlyIssue issue);
void Progress(int percentage, string currentOperation = null); void Progress(int percentage, string currentOperation = null);
void UpdateDetailTimelineRecord(TimelineRecord record); void UpdateDetailTimelineRecord(TimelineRecord record);
@@ -126,8 +126,10 @@ namespace GitHub.Runner.Worker
private readonly TimelineRecord _record = new(); private readonly TimelineRecord _record = new();
private readonly Dictionary<Guid, TimelineRecord> _detailRecords = new(); private readonly Dictionary<Guid, TimelineRecord> _detailRecords = new();
private readonly List<IReadOnlyIssue> _embeddedIssueCollector;
private readonly object _loggerLock = new(); private readonly object _loggerLock = new();
private readonly object _matchersLock = new(); private readonly object _matchersLock = new();
private readonly ExecutionContext _parentExecutionContext;
private event OnMatcherChanged _onMatcherChanged; private event OnMatcherChanged _onMatcherChanged;
@@ -135,7 +137,6 @@ namespace GitHub.Runner.Worker
private IPagingLogger _logger; private IPagingLogger _logger;
private IJobServerQueue _jobServerQueue; private IJobServerQueue _jobServerQueue;
private ExecutionContext _parentExecutionContext;
private Guid _mainTimelineId; private Guid _mainTimelineId;
private Guid _detailTimelineId; private Guid _detailTimelineId;
@@ -149,6 +150,29 @@ namespace GitHub.Runner.Worker
private long _totalThrottlingDelayInMilliseconds = 0; private long _totalThrottlingDelayInMilliseconds = 0;
private bool _stepTelemetryPublished = false; private bool _stepTelemetryPublished = false;
public ExecutionContext()
: this(null, false)
{
}
private ExecutionContext(ExecutionContext parent, bool embedded)
{
if (embedded)
{
ArgUtil.NotNull(parent, nameof(parent));
}
_parentExecutionContext = parent;
this.IsEmbedded = embedded;
this.StepTelemetry = new ActionsStepTelemetry
{
IsEmbedded = embedded
};
//Embedded Execution Contexts pseudo-inherit their parent's embeddedIssueCollector.
_embeddedIssueCollector = embedded ? parent._embeddedIssueCollector : new();
}
public Guid Id => _record.Id; public Guid Id => _record.Id;
public Guid EmbeddedId { get; private set; } public Guid EmbeddedId { get; private set; }
public string ScopeName { get; private set; } public string ScopeName { get; private set; }
@@ -161,7 +185,7 @@ namespace GitHub.Runner.Worker
public Dictionary<string, VariableValue> JobOutputs { get; private set; } public Dictionary<string, VariableValue> JobOutputs { get; private set; }
public ActionsEnvironmentReference ActionsEnvironment { get; private set; } public ActionsEnvironmentReference ActionsEnvironment { get; private set; }
public ActionsStepTelemetry StepTelemetry { get; } = new ActionsStepTelemetry(); public ActionsStepTelemetry StepTelemetry { get; private init; }
public DictionaryContextData ExpressionValues { get; } = new DictionaryContextData(); public DictionaryContextData ExpressionValues { get; } = new DictionaryContextData();
public IList<IFunctionInfo> ExpressionFunctions { get; } = new List<IFunctionInfo>(); public IList<IFunctionInfo> ExpressionFunctions { get; } = new List<IFunctionInfo>();
@@ -186,7 +210,7 @@ namespace GitHub.Runner.Worker
// An embedded execution context shares the same record ID, record name, and logger // An embedded execution context shares the same record ID, record name, and logger
// as its enclosing execution context. // as its enclosing execution context.
public bool IsEmbedded { get; private set; } public bool IsEmbedded { get; private init; }
public TaskResult? Result public TaskResult? Result
{ {
@@ -321,7 +345,7 @@ namespace GitHub.Runner.Worker
{ {
Trace.Entering(); Trace.Entering();
var child = new ExecutionContext(); var child = new ExecutionContext(this, isEmbedded);
child.Initialize(HostContext); child.Initialize(HostContext);
child.Global = Global; child.Global = Global;
child.ScopeName = scopeName; child.ScopeName = scopeName;
@@ -346,7 +370,6 @@ namespace GitHub.Runner.Worker
child.ExpressionFunctions.Add(item); child.ExpressionFunctions.Add(item);
} }
child._cancellationTokenSource = cancellationTokenSource ?? new CancellationTokenSource(); child._cancellationTokenSource = cancellationTokenSource ?? new CancellationTokenSource();
child._parentExecutionContext = this;
child.EchoOnActionCommand = EchoOnActionCommand; child.EchoOnActionCommand = EchoOnActionCommand;
if (recordOrder != null) if (recordOrder != null)
@@ -367,11 +390,9 @@ namespace GitHub.Runner.Worker
child._logger.Setup(_mainTimelineId, recordId); child._logger.Setup(_mainTimelineId, recordId);
} }
child.IsEmbedded = isEmbedded;
child.StepTelemetry.StepId = recordId; child.StepTelemetry.StepId = recordId;
child.StepTelemetry.Stage = stage.ToString(); child.StepTelemetry.Stage = stage.ToString();
child.StepTelemetry.IsEmbedded = isEmbedded; child.StepTelemetry.StepContextName = child.GetFullyQualifiedContextName();
child.StepTelemetry.StepContextName = child.GetFullyQualifiedContextName(); ;
return child; return child;
} }
@@ -413,13 +434,24 @@ namespace GitHub.Runner.Worker
this.Warning($"The job has experienced {TimeSpan.FromMilliseconds(_totalThrottlingDelayInMilliseconds).TotalSeconds} seconds total delay caused by server throttling."); this.Warning($"The job has experienced {TimeSpan.FromMilliseconds(_totalThrottlingDelayInMilliseconds).TotalSeconds} seconds total delay caused by server throttling.");
} }
DateTime now = DateTime.UtcNow;
_record.CurrentOperation = currentOperation ?? _record.CurrentOperation; _record.CurrentOperation = currentOperation ?? _record.CurrentOperation;
_record.ResultCode = resultCode ?? _record.ResultCode; _record.ResultCode = resultCode ?? _record.ResultCode;
_record.FinishTime = DateTime.UtcNow; _record.FinishTime = now;
_record.PercentComplete = 100; _record.PercentComplete = 100;
_record.Result = _record.Result ?? TaskResult.Succeeded; _record.Result = _record.Result ?? TaskResult.Succeeded;
_record.State = TimelineRecordState.Completed; _record.State = TimelineRecordState.Completed;
// Before our main timeline's final QueueTimelineRecordUpdate,
// inject any issues collected by embedded ExecutionContexts.
if (!this.IsEmbedded)
{
foreach (var issue in _embeddedIssueCollector)
{
AddIssue(issue);
}
}
_jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record); _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
// complete all detail timeline records. // complete all detail timeline records.
@@ -427,7 +459,7 @@ namespace GitHub.Runner.Worker
{ {
foreach (var record in _detailRecords) foreach (var record in _detailRecords)
{ {
record.Value.FinishTime = record.Value.FinishTime ?? DateTime.UtcNow; record.Value.FinishTime = record.Value.FinishTime ?? now;
record.Value.PercentComplete = record.Value.PercentComplete ?? 100; record.Value.PercentComplete = record.Value.PercentComplete ?? 100;
record.Value.Result = record.Value.Result ?? TaskResult.Succeeded; record.Value.Result = record.Value.Result ?? TaskResult.Succeeded;
record.Value.State = TimelineRecordState.Completed; record.Value.State = TimelineRecordState.Completed;
@@ -438,17 +470,6 @@ namespace GitHub.Runner.Worker
PublishStepTelemetry(); PublishStepTelemetry();
var stepResult = new StepResult();
stepResult.ExternalID = _record.Id;
stepResult.Conclusion = _record.Result ?? TaskResult.Succeeded;
stepResult.Status = _record.State;
stepResult.Number = _record.Order;
stepResult.Name = _record.Name;
stepResult.StartedAt = _record.StartTime;
stepResult.CompletedAt = _record.FinishTime;
Global.StepsResult.Add(stepResult);
if (Root != this) if (Root != this)
{ {
// only dispose TokenSource for step level ExecutionContext // only dispose TokenSource for step level ExecutionContext
@@ -558,77 +579,90 @@ namespace GitHub.Runner.Worker
_jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record); _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
} }
// This is not thread safe, the caller need to take lock before calling issue() // This is not thread safe, the caller needs to take lock before calling issue()
public void AddIssue(Issue issue, string logMessage = null) public IReadOnlyIssue CreateIssue(IssueType issueType, string rawMessage, IssueMetadata metadata, bool writeToLog)
{
string refinedMessage = PrimitiveExtensions.TrimExcess(HostContext.SecretMasker.MaskSecrets(rawMessage), _maxIssueMessageLength);
var result = new Issue() {
Type = issueType,
Message = refinedMessage,
};
if (metadata != null)
{
result.Category = metadata.Category;
result.IsInfrastructureIssue = metadata.IsInfrastructureIssue;
foreach (var kvp in metadata.Data)
{
result[kvp.Key] = kvp.Value;
}
}
// It's important to keep track of the step number (key:stepNumber) and the line number (key:logFileLineNumber) of every issue that gets logged.
// Actions UI from the run summary page use both values to easily link to an exact locations in logs where annotations originate from.
if (_record.Order != null)
{
result["stepNumber"] = _record.Order.ToString();
}
string wellKnownTag = null;
Int32? previousCountForIssueType = null;
switch (issueType)
{
case IssueType.Error:
wellKnownTag = WellKnownTags.Error;
previousCountForIssueType = _record.ErrorCount++;
break;
case IssueType.Warning:
wellKnownTag = WellKnownTags.Warning;
previousCountForIssueType = _record.WarningCount++;
break;
case IssueType.Notice:
wellKnownTag = WellKnownTags.Notice;
previousCountForIssueType = _record.NoticeCount++;
break;
}
if (!string.IsNullOrEmpty(wellKnownTag))
{
if (writeToLog)
{
//Note that ::Write() has it's own secret masking logic
string logText = metadata?.LogMessageOverride ?? result.Message;
if (!string.IsNullOrEmpty(logText))
{
long logLineNumber = Write(wellKnownTag, logText);
result["logFileLineNumber"] = logLineNumber.ToString();
}
}
if (previousCountForIssueType.GetValueOrDefault(0) < _maxIssueCount)
{
_record.Issues.Add(result);
}
}
return result;
}
// This is not thread safe, the caller needs to take lock before calling issue()
public void AddIssue(IReadOnlyIssue issue)
{ {
ArgUtil.NotNull(issue, nameof(issue)); ArgUtil.NotNull(issue, nameof(issue));
if (string.IsNullOrEmpty(logMessage)) // Embedded ExecutionContexts (a.k.a. Composite actions) should never upload a timeline record to the server.
// Instead, we store processed issues on a shared (psuedo-inherited) list (belonging to the closest
// non-embedded ancestor ExecutionContext) so that they can be processed when that ancestor completes.
if (this.IsEmbedded)
{ {
logMessage = issue.Message; _embeddedIssueCollector.Add(issue);
} }
else
issue.Message = HostContext.SecretMasker.MaskSecrets(issue.Message);
if (issue.Message.Length > _maxIssueMessageLength)
{ {
issue.Message = issue.Message[.._maxIssueMessageLength];
}
// Tracking the line number (logFileLineNumber) and step number (stepNumber) for each issue that gets created
// Actions UI from the run summary page use both values to easily link to an exact locations in logs where annotations originate from
if (_record.Order != null)
{
issue.Data["stepNumber"] = _record.Order.ToString();
}
if (issue.Type == IssueType.Error)
{
if (!string.IsNullOrEmpty(logMessage))
{
long logLineNumber = Write(WellKnownTags.Error, logMessage);
issue.Data["logFileLineNumber"] = logLineNumber.ToString();
}
if (_record.ErrorCount < _maxIssueCount)
{
_record.Issues.Add(issue);
}
_record.ErrorCount++;
}
else if (issue.Type == IssueType.Warning)
{
if (!string.IsNullOrEmpty(logMessage))
{
long logLineNumber = Write(WellKnownTags.Warning, logMessage);
issue.Data["logFileLineNumber"] = logLineNumber.ToString();
}
if (_record.WarningCount < _maxIssueCount)
{
_record.Issues.Add(issue);
}
_record.WarningCount++;
}
else if (issue.Type == IssueType.Notice)
{
if (!string.IsNullOrEmpty(logMessage))
{
long logLineNumber = Write(WellKnownTags.Notice, logMessage);
issue.Data["logFileLineNumber"] = logLineNumber.ToString();
}
if (_record.NoticeCount < _maxIssueCount)
{
_record.Issues.Add(issue);
}
_record.NoticeCount++;
}
_jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record); _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record);
} }
}
public void UpdateDetailTimelineRecord(TimelineRecord record) public void UpdateDetailTimelineRecord(TimelineRecord record)
{ {
@@ -722,9 +756,6 @@ namespace GitHub.Runner.Worker
// ActionsStepTelemetry for entire job // ActionsStepTelemetry for entire job
Global.StepsTelemetry = new List<ActionsStepTelemetry>(); Global.StepsTelemetry = new List<ActionsStepTelemetry>();
// Steps results for entire job
Global.StepsResult = new List<StepResult>();
// Job Outputs // Job Outputs
JobOutputs = new Dictionary<string, VariableValue>(StringComparer.OrdinalIgnoreCase); JobOutputs = new Dictionary<string, VariableValue>(StringComparer.OrdinalIgnoreCase);
@@ -862,18 +893,6 @@ namespace GitHub.Runner.Worker
_jobServerQueue.QueueFileUpload(_mainTimelineId, _record.Id, type, name, filePath, deleteSource: false); _jobServerQueue.QueueFileUpload(_mainTimelineId, _record.Id, type, name, filePath, deleteSource: false);
} }
public void QueueSummaryFile(string name, string filePath, Guid stepRecordId)
{
ArgUtil.NotNullOrEmpty(name, nameof(name));
ArgUtil.NotNullOrEmpty(filePath, nameof(filePath));
if (!File.Exists(filePath))
{
throw new FileNotFoundException($"Can't upload (name:{name}) file: {filePath}. File does not exist.");
}
_jobServerQueue.QueueResultsUpload(stepRecordId, name, filePath, ChecksAttachmentType.StepSummary, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0);
}
// Add OnMatcherChanged // Add OnMatcherChanged
public void Add(OnMatcherChanged handler) public void Add(OnMatcherChanged handler)
{ {
@@ -1003,16 +1022,7 @@ namespace GitHub.Runner.Worker
if ((issue.Type == IssueType.Error || issue.Type == IssueType.Warning) && if ((issue.Type == IssueType.Error || issue.Type == IssueType.Warning) &&
!string.IsNullOrEmpty(issue.Message)) !string.IsNullOrEmpty(issue.Message))
{ {
string issueTelemetry; string issueTelemetry = PrimitiveExtensions.TrimExcess(issue.Message, _maxIssueMessageLengthInTelemetry);
if (issue.Message.Length > _maxIssueMessageLengthInTelemetry)
{
issueTelemetry = $"{issue.Message[.._maxIssueMessageLengthInTelemetry]}";
}
else
{
issueTelemetry = issue.Message;
}
StepTelemetry.ErrorMessages.Add(issueTelemetry); StepTelemetry.ErrorMessages.Add(issueTelemetry);
// Only send over the first 3 issues to avoid sending too much data. // Only send over the first 3 issues to avoid sending too much data.
@@ -1186,19 +1196,23 @@ namespace GitHub.Runner.Worker
// Do not add a format string overload. See comment on ExecutionContext.Write(). // Do not add a format string overload. See comment on ExecutionContext.Write().
public static void Error(this IExecutionContext context, string message) public static void Error(this IExecutionContext context, string message)
{ {
context.AddIssue(new Issue() { Type = IssueType.Error, Message = message }); var issue = context.CreateIssue(IssueType.Error, message, null, true);
context.AddIssue(issue);
} }
// Do not add a format string overload. See comment on ExecutionContext.Write(). // Do not add a format string overload. See comment on ExecutionContext.Write().
public static void InfrastructureError(this IExecutionContext context, string message) public static void InfrastructureError(this IExecutionContext context, string message)
{ {
context.AddIssue(new Issue() { Type = IssueType.Error, Message = message, IsInfrastructureIssue = true }); var metadata = new IssueMetadata(null, true, null, Enumerable.Empty<KeyValuePair<string, string>>());
var issue = context.CreateIssue(IssueType.Error, message, metadata, true);
context.AddIssue(issue);
} }
// Do not add a format string overload. See comment on ExecutionContext.Write(). // Do not add a format string overload. See comment on ExecutionContext.Write().
public static void Warning(this IExecutionContext context, string message) public static void Warning(this IExecutionContext context, string message)
{ {
context.AddIssue(new Issue() { Type = IssueType.Warning, Message = message }); var issue = context.CreateIssue(IssueType.Warning, message, null, true);
context.AddIssue(issue);
} }
// Do not add a format string overload. See comment on ExecutionContext.Write(). // Do not add a format string overload. See comment on ExecutionContext.Write().

View File

@@ -29,8 +29,6 @@ namespace GitHub.Runner.Worker.Expressions
githubContext.TryGetValue(PipelineTemplateConstants.Workspace, out var workspace); githubContext.TryGetValue(PipelineTemplateConstants.Workspace, out var workspace);
var workspaceData = workspace as StringContextData; var workspaceData = workspace as StringContextData;
ArgUtil.NotNull(workspaceData, nameof(workspaceData)); ArgUtil.NotNull(workspaceData, nameof(workspaceData));
var executionContext = templateContext.State[nameof(IExecutionContext)] as IExecutionContext;
ArgUtil.NotNull(executionContext, nameof(executionContext));
string githubWorkspace = workspaceData.Value; string githubWorkspace = workspaceData.Value;
bool followSymlink = false; bool followSymlink = false;
@@ -66,15 +64,7 @@ namespace GitHub.Runner.Worker.Expressions
string runnerRoot = new DirectoryInfo(binDir).Parent.FullName; string runnerRoot = new DirectoryInfo(binDir).Parent.FullName;
string node = Path.Combine(runnerRoot, "externals", NodeUtil.GetInternalNodeVersion(), "bin", $"node{IOUtil.ExeExtension}"); string node = Path.Combine(runnerRoot, "externals", NodeUtil.GetInternalNodeVersion(), "bin", $"node{IOUtil.ExeExtension}");
//Feature flag to fetch a new version of hashFiles script string hashFilesScript = Path.Combine(binDir, "hashFiles");
string hashFilesScript = string.Empty;
var isGlobHashFilesEnabled = executionContext.Global.Variables.GetBoolean("DistributedTask.UseGlobHashFiles") ?? false;
if (isGlobHashFilesEnabled)
{
hashFilesScript = Path.Combine(binDir, "hashFilesV2");
}
hashFilesScript = Path.Combine(binDir, "hashFiles");
var hashResult = string.Empty; var hashResult = string.Empty;
var p = new ProcessInvoker(new HashFilesTrace(context.Trace)); var p = new ProcessInvoker(new HashFilesTrace(context.Trace));
p.ErrorDataReceived += ((_, data) => p.ErrorDataReceived += ((_, data) =>

View File

@@ -211,16 +211,6 @@ namespace GitHub.Runner.Worker
Trace.Info($"Queueing file ({filePath}) for attachment upload ({attachmentName})"); Trace.Info($"Queueing file ({filePath}) for attachment upload ({attachmentName})");
// Attachments must be added to the parent context (job), not the current context (step) // Attachments must be added to the parent context (job), not the current context (step)
context.Root.QueueAttachFile(ChecksAttachmentType.StepSummary, attachmentName, scrubbedFilePath); context.Root.QueueAttachFile(ChecksAttachmentType.StepSummary, attachmentName, scrubbedFilePath);
// Dual upload the same files to Results Service
context.Global.Variables.TryGetValue("system.github.results_endpoint", out string resultsReceiverEndpoint);
if (resultsReceiverEndpoint != null)
{
Trace.Info($"Queueing results file ({filePath}) for attachment upload ({attachmentName})");
var stepId = context.Id;
// Attachments must be added to the parent context (job), not the current context (step)
context.Root.QueueSummaryFile(attachmentName, scrubbedFilePath, stepId);
}
} }
catch (Exception e) catch (Exception e)
{ {

View File

@@ -1,6 +1,5 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using GitHub.Actions.RunService.WebApi;
using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common.Util; using GitHub.Runner.Common.Util;
using GitHub.Runner.Worker.Container; using GitHub.Runner.Worker.Container;
@@ -17,7 +16,6 @@ namespace GitHub.Runner.Worker
public IList<String> FileTable { get; set; } public IList<String> FileTable { get; set; }
public IDictionary<String, IDictionary<String, String>> JobDefaults { get; set; } public IDictionary<String, IDictionary<String, String>> JobDefaults { get; set; }
public List<ActionsStepTelemetry> StepsTelemetry { get; set; } public List<ActionsStepTelemetry> StepsTelemetry { get; set; }
public List<StepResult> StepsResult { get; set; }
public List<JobTelemetry> JobTelemetry { get; set; } public List<JobTelemetry> JobTelemetry { get; set; }
public TaskOrchestrationPlanReference Plan { get; set; } public TaskOrchestrationPlanReference Plan { get; set; }
public List<string> PrependPath { get; set; } public List<string> PrependPath { get; set; }

View File

@@ -294,7 +294,7 @@ namespace GitHub.Runner.Worker.Handlers
// Evaluation error // Evaluation error
Trace.Info("Caught exception from expression for embedded step.env"); Trace.Info("Caught exception from expression for embedded step.env");
step.ExecutionContext.Error(ex); step.ExecutionContext.Error(ex);
step.ExecutionContext.Complete(TaskResult.Failed); SetStepConclusion(step, TaskResult.Failed);
} }
// Register Callback // Register Callback

View File

@@ -38,17 +38,8 @@ namespace GitHub.Runner.Worker.Handlers
// Update the env dictionary. // Update the env dictionary.
AddInputsToEnvironment(); AddInputsToEnvironment();
IDockerCommandManager dockerManager = null; var dockerManager = HostContext.GetService<IDockerCommandManager>();
IContainerHookManager containerHookManager = null; var containerHookManager = HostContext.GetService<IContainerHookManager>();
if (FeatureManager.IsContainerHooksEnabled(ExecutionContext.Global.Variables))
{
containerHookManager = HostContext.GetService<IContainerHookManager>();
}
else
{
dockerManager = HostContext.GetService<IDockerCommandManager>();
}
string dockerFile = null; string dockerFile = null;
// container image haven't built/pull // container image haven't built/pull

View File

@@ -4,6 +4,7 @@ using System.Globalization;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common; using GitHub.Runner.Common;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;
using GitHub.Runner.Worker.Container; using GitHub.Runner.Worker.Container;
@@ -97,7 +98,7 @@ namespace GitHub.Runner.Worker.Handlers
var matchers = _matchers; var matchers = _matchers;
// Strip color codes // Strip color codes
var stripped = line.Contains(_colorCodePrefix) ? _colorCodeRegex.Replace(line, string.Empty) : line; var refinedLine = line.Contains(_colorCodePrefix) ? _colorCodeRegex.Replace(line, string.Empty) : line;
foreach (var matcher in matchers) foreach (var matcher in matchers)
{ {
@@ -107,8 +108,7 @@ namespace GitHub.Runner.Worker.Handlers
// Match // Match
try try
{ {
match = matcher.Match(stripped); match = matcher.Match(refinedLine);
break; break;
} }
catch (RegexMatchTimeoutException ex) catch (RegexMatchTimeoutException ex)
@@ -116,7 +116,7 @@ namespace GitHub.Runner.Worker.Handlers
if (attempt < _maxAttempts) if (attempt < _maxAttempts)
{ {
// Debug // Debug
_executionContext.Debug($"Timeout processing issue matcher '{matcher.Owner}' against line '{stripped}'. Exception: {ex.ToString()}"); _executionContext.Debug($"Timeout processing issue matcher '{matcher.Owner}' against line '{refinedLine}'. Exception: {ex}");
} }
else else
{ {
@@ -139,12 +139,10 @@ namespace GitHub.Runner.Worker.Handlers
// Convert to issue // Convert to issue
var issue = ConvertToIssue(match); var issue = ConvertToIssue(match);
if (issue != null) if (issue != null)
{ {
// Log issue // Log issue
_executionContext.AddIssue(issue, stripped); _executionContext.AddIssue(issue);
return; return;
} }
} }
@@ -196,7 +194,7 @@ namespace GitHub.Runner.Worker.Handlers
} }
} }
private DTWebApi.Issue ConvertToIssue(IssueMatch match) private DTWebApi.IReadOnlyIssue ConvertToIssue(IssueMatch match)
{ {
// Validate the message // Validate the message
if (string.IsNullOrWhiteSpace(match.Message)) if (string.IsNullOrWhiteSpace(match.Message))
@@ -225,18 +223,14 @@ namespace GitHub.Runner.Worker.Handlers
return null; return null;
} }
var issue = new DTWebApi.Issue var issueData = new Dictionary<string, string>();
{
Message = match.Message,
Type = issueType,
};
// Line // Line
if (!string.IsNullOrEmpty(match.Line)) if (!string.IsNullOrEmpty(match.Line))
{ {
if (int.TryParse(match.Line, NumberStyles.None, CultureInfo.InvariantCulture, out var line)) if (int.TryParse(match.Line, NumberStyles.None, CultureInfo.InvariantCulture, out var line))
{ {
issue.Data["line"] = line.ToString(CultureInfo.InvariantCulture); issueData["line"] = line.ToString(CultureInfo.InvariantCulture);
} }
else else
{ {
@@ -249,7 +243,7 @@ namespace GitHub.Runner.Worker.Handlers
{ {
if (int.TryParse(match.Column, NumberStyles.None, CultureInfo.InvariantCulture, out var column)) if (int.TryParse(match.Column, NumberStyles.None, CultureInfo.InvariantCulture, out var column))
{ {
issue.Data["col"] = column.ToString(CultureInfo.InvariantCulture); issueData["col"] = column.ToString(CultureInfo.InvariantCulture);
} }
else else
{ {
@@ -260,7 +254,7 @@ namespace GitHub.Runner.Worker.Handlers
// Code // Code
if (!string.IsNullOrWhiteSpace(match.Code)) if (!string.IsNullOrWhiteSpace(match.Code))
{ {
issue.Data["code"] = match.Code.Trim(); issueData["code"] = match.Code.Trim();
} }
// File // File
@@ -312,7 +306,7 @@ namespace GitHub.Runner.Worker.Handlers
var relativePath = file.Substring(repositoryPath.Length).TrimStart(Path.DirectorySeparatorChar); var relativePath = file.Substring(repositoryPath.Length).TrimStart(Path.DirectorySeparatorChar);
// Prefer `/` on all platforms // Prefer `/` on all platforms
issue.Data["file"] = relativePath.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); issueData["file"] = relativePath.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
} }
else else
{ {
@@ -327,9 +321,11 @@ namespace GitHub.Runner.Worker.Handlers
} }
catch (Exception ex) catch (Exception ex)
{ {
_executionContext.Debug($"Dropping file value '{match.File}' and fromPath value '{match.FromPath}'. Exception during validation: {ex.ToString()}"); _executionContext.Debug($"Dropping file value '{match.File}' and fromPath value '{match.FromPath}'. Exception during validation: {ex}");
} }
var metadata = new IssueMetadata(null, false, match.SourceText, issueData);
var issue = _executionContext.CreateIssue(issueType, match.Message, metadata, true);
return issue; return issue;
} }

View File

@@ -8,6 +8,31 @@ namespace GitHub.Runner.Worker
{ {
public delegate void OnMatcherChanged(object sender, MatcherChangedEventArgs e); public delegate void OnMatcherChanged(object sender, MatcherChangedEventArgs e);
public sealed class IssueMetadata
{
public IssueMetadata(string key, string value)
: this(null, false, null, new []{ KeyValuePair.Create(key, value) })
{
}
public IssueMetadata(string category, bool infrastructureIssue, string logMessageOverride, IEnumerable<KeyValuePair<string, string>> data)
{
this.Category = category;
this.IsInfrastructureIssue = infrastructureIssue;
this.LogMessageOverride = logMessageOverride;
// Close-over the incoming IEnumerable to force immediate evaluation.
var empty = Enumerable.Empty<KeyValuePair<string, string>>();
this.Data = new Dictionary<string, string>(data ?? empty, StringComparer.OrdinalIgnoreCase);
}
public readonly string Category;
public readonly bool IsInfrastructureIssue;
public readonly string LogMessageOverride;
public readonly IEnumerable<KeyValuePair<string, string>> Data;
}
public sealed class MatcherChangedEventArgs : EventArgs public sealed class MatcherChangedEventArgs : EventArgs
{ {
public MatcherChangedEventArgs(IssueMatcherConfig config) public MatcherChangedEventArgs(IssueMatcherConfig config)
@@ -69,7 +94,7 @@ namespace GitHub.Runner.Worker
if (regexMatch.Success) if (regexMatch.Success)
{ {
return new IssueMatch(null, pattern, regexMatch.Groups, DefaultSeverity); return new IssueMatch(line, null, pattern, regexMatch.Groups, DefaultSeverity);
} }
return null; return null;
@@ -110,13 +135,13 @@ namespace GitHub.Runner.Worker
} }
// Return // Return
return new IssueMatch(runningMatch, pattern, regexMatch.Groups, DefaultSeverity); return new IssueMatch(line, runningMatch, pattern, regexMatch.Groups, DefaultSeverity);
} }
// Not the last pattern // Not the last pattern
else else
{ {
// Store the match // Store the match
_state[i] = new IssueMatch(runningMatch, pattern, regexMatch.Groups); _state[i] = new IssueMatch(line, runningMatch, pattern, regexMatch.Groups);
} }
} }
// Not matched // Not matched
@@ -184,8 +209,9 @@ namespace GitHub.Runner.Worker
public sealed class IssueMatch public sealed class IssueMatch
{ {
public IssueMatch(IssueMatch runningMatch, IssuePattern pattern, GroupCollection groups, string defaultSeverity = null) public IssueMatch(string sourceText, IssueMatch runningMatch, IssuePattern pattern, GroupCollection groups, string defaultSeverity = null)
{ {
SourceText = sourceText;
File = runningMatch?.File ?? GetValue(groups, pattern.File); File = runningMatch?.File ?? GetValue(groups, pattern.File);
Line = runningMatch?.Line ?? GetValue(groups, pattern.Line); Line = runningMatch?.Line ?? GetValue(groups, pattern.Line);
Column = runningMatch?.Column ?? GetValue(groups, pattern.Column); Column = runningMatch?.Column ?? GetValue(groups, pattern.Column);
@@ -200,6 +226,8 @@ namespace GitHub.Runner.Worker
} }
} }
public string SourceText { get; }
public string File { get; } public string File { get; }
public string Line { get; } public string Line { get; }

View File

@@ -306,13 +306,13 @@ namespace GitHub.Runner.Worker
} }
} }
actionRunner.EvaluateDisplayName(contextData, context, out _); actionRunner.TryEvaluateDisplayName(contextData, context);
jobSteps.Add(actionRunner); jobSteps.Add(actionRunner);
if (prepareResult.PreStepTracker.TryGetValue(step.Id, out var preStep)) if (prepareResult.PreStepTracker.TryGetValue(step.Id, out var preStep))
{ {
Trace.Info($"Adding pre-{action.DisplayName}."); Trace.Info($"Adding pre-{action.DisplayName}.");
preStep.EvaluateDisplayName(contextData, context, out _); preStep.TryEvaluateDisplayName(contextData, context);
preStep.DisplayName = $"Pre {preStep.DisplayName}"; preStep.DisplayName = $"Pre {preStep.DisplayName}";
preJobSteps.Add(preStep); preJobSteps.Add(preStep);
} }
@@ -321,10 +321,7 @@ namespace GitHub.Runner.Worker
if (message.Variables.TryGetValue("system.workflowFileFullPath", out VariableValue workflowFileFullPath)) if (message.Variables.TryGetValue("system.workflowFileFullPath", out VariableValue workflowFileFullPath))
{ {
var usesLogText = $"Uses: {workflowFileFullPath.Value}"; context.Output($"Uses: {workflowFileFullPath.Value}");
var reference = GetWorkflowReference(message.Variables);
context.Output(usesLogText + reference);
if (message.ContextData.TryGetValue("inputs", out var pipelineContextData)) if (message.ContextData.TryGetValue("inputs", out var pipelineContextData))
{ {
var inputs = pipelineContextData.AssertDictionary("inputs"); var inputs = pipelineContextData.AssertDictionary("inputs");
@@ -338,12 +335,12 @@ namespace GitHub.Runner.Worker
context.Output("##[endgroup]"); context.Output("##[endgroup]");
} }
} }
}
if (!string.IsNullOrWhiteSpace(message.JobDisplayName)) if (!string.IsNullOrWhiteSpace(message.JobDisplayName))
{ {
context.Output($"Complete job name: {message.JobDisplayName}"); context.Output($"Complete job name: {message.JobDisplayName}");
} }
}
var intraActionStates = new Dictionary<Guid, Dictionary<string, string>>(); var intraActionStates = new Dictionary<Guid, Dictionary<string, string>>();
foreach (var preStep in prepareResult.PreStepTracker) foreach (var preStep in prepareResult.PreStepTracker)
@@ -455,24 +452,6 @@ namespace GitHub.Runner.Worker
} }
} }
private string GetWorkflowReference(IDictionary<string, VariableValue> variables)
{
var reference = "";
if (variables.TryGetValue("system.workflowFileSha", out VariableValue workflowFileSha))
{
if (variables.TryGetValue("system.workflowFileRef", out VariableValue workflowFileRef)
&& !string.IsNullOrEmpty(workflowFileRef.Value))
{
reference += $"@{workflowFileRef.Value} ({workflowFileSha.Value})";
}
else
{
reference += $"@{workflowFileSha.Value}";
}
}
return reference;
}
public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc) public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc)
{ {
Trace.Entering(); Trace.Entering();
@@ -681,8 +660,9 @@ namespace GitHub.Runner.Worker
var freeSpaceInMB = driveInfo.AvailableFreeSpace / 1024 / 1024; var freeSpaceInMB = driveInfo.AvailableFreeSpace / 1024 / 1024;
if (freeSpaceInMB < lowDiskSpaceThreshold) if (freeSpaceInMB < lowDiskSpaceThreshold)
{ {
var issue = new Issue() { Type = IssueType.Warning, Message = $"You are running out of disk space. The runner will stop working when the machine runs out of disk space. Free space left: {freeSpaceInMB} MB" }; var message = $"You are running out of disk space. The runner will stop working when the machine runs out of disk space. Free space left: {freeSpaceInMB} MB";
issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.LowDiskSpace; var metadata = new IssueMetadata(Constants.Runner.InternalTelemetryIssueDataKey, Constants.Runner.LowDiskSpace);
var issue = context.CreateIssue(IssueType.Warning, message, metadata, true);
context.AddIssue(issue); context.AddIssue(issue);
return; return;
} }

View File

@@ -1,7 +1,6 @@
using System; using System;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.DistributedTask.ObjectTemplating.Tokens; using GitHub.DistributedTask.ObjectTemplating.Tokens;
using GitHub.DistributedTask.Pipelines.ContextData;
namespace GitHub.Runner.Worker namespace GitHub.Runner.Worker
{ {
@@ -33,18 +32,5 @@ namespace GitHub.Runner.Worker
{ {
await _runAsync(ExecutionContext, _data); await _runAsync(ExecutionContext, _data);
} }
public bool TryUpdateDisplayName(out bool updated)
{
updated = false;
return !string.IsNullOrEmpty(this.DisplayName);
}
public bool EvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context, out bool updated)
{
updated = false;
return !string.IsNullOrEmpty(this.DisplayName);
}
} }
} }

View File

@@ -6,7 +6,7 @@ using System.Net.Http;
using System.Text; using System.Text;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common; using GitHub.Runner.Common;
using GitHub.Runner.Common.Util; using GitHub.Runner.Common.Util;
@@ -20,7 +20,7 @@ namespace GitHub.Runner.Worker
[ServiceLocator(Default = typeof(JobRunner))] [ServiceLocator(Default = typeof(JobRunner))]
public interface IJobRunner : IRunnerService public interface IJobRunner : IRunnerService
{ {
Task<TaskResult> RunAsync(AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken); Task<TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken);
} }
public sealed class JobRunner : RunnerService, IJobRunner public sealed class JobRunner : RunnerService, IJobRunner
@@ -29,7 +29,7 @@ namespace GitHub.Runner.Worker
private RunnerSettings _runnerSettings; private RunnerSettings _runnerSettings;
private ITempDirectoryManager _tempDirectoryManager; private ITempDirectoryManager _tempDirectoryManager;
public async Task<TaskResult> RunAsync(AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken) public async Task<TaskResult> RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken)
{ {
// Validate parameters. // Validate parameters.
Trace.Entering(); Trace.Entering();
@@ -40,18 +40,9 @@ namespace GitHub.Runner.Worker
Trace.Info("Job ID {0}", message.JobId); Trace.Info("Job ID {0}", message.JobId);
DateTime jobStartTimeUtc = DateTime.UtcNow; DateTime jobStartTimeUtc = DateTime.UtcNow;
IRunnerService server = null;
ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
if (MessageUtil.IsRunServiceJob(message.MessageType))
{
var runServer = HostContext.GetService<IRunServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
await runServer.ConnectAsync(systemConnection.Url, jobServerCredential);
server = runServer;
}
else
{
// Setup the job server and job server queue. // Setup the job server and job server queue.
var jobServer = HostContext.GetService<IJobServer>(); var jobServer = HostContext.GetService<IJobServer>();
VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection);
@@ -64,10 +55,6 @@ namespace GitHub.Runner.Worker
await jobServer.ConnectAsync(jobConnection); await jobServer.ConnectAsync(jobConnection);
_jobServerQueue.Start(message); _jobServerQueue.Start(message);
server = jobServer;
}
HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}"); HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}");
IExecutionContext jobContext = null; IExecutionContext jobContext = null;
@@ -97,7 +84,8 @@ namespace GitHub.Runner.Worker
default: default:
throw new ArgumentException(HostContext.RunnerShutdownReason.ToString(), nameof(HostContext.RunnerShutdownReason)); throw new ArgumentException(HostContext.RunnerShutdownReason.ToString(), nameof(HostContext.RunnerShutdownReason));
} }
jobContext.AddIssue(new Issue() { Type = IssueType.Error, Message = errorMessage }); var issue = jobContext.CreateIssue(IssueType.Error, errorMessage, null, true);
jobContext.AddIssue(issue);
}); });
// Validate directory permissions. // Validate directory permissions.
@@ -112,7 +100,7 @@ namespace GitHub.Runner.Worker
{ {
Trace.Error(ex); Trace.Error(ex);
jobContext.Error(ex); jobContext.Error(ex);
return await CompleteJobAsync(server, jobContext, message, TaskResult.Failed); return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed);
} }
if (jobContext.Global.WriteDebug) if (jobContext.Global.WriteDebug)
@@ -149,7 +137,7 @@ namespace GitHub.Runner.Worker
// don't log error issue to job ExecutionContext, since server owns the job level issue // don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job is cancelled during initialize."); Trace.Error($"Job is cancelled during initialize.");
Trace.Error($"Caught exception: {ex}"); Trace.Error($"Caught exception: {ex}");
return await CompleteJobAsync(server, jobContext, message, TaskResult.Canceled); return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled);
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -157,7 +145,7 @@ namespace GitHub.Runner.Worker
// don't log error issue to job ExecutionContext, since server owns the job level issue // don't log error issue to job ExecutionContext, since server owns the job level issue
Trace.Error($"Job initialize failed."); Trace.Error($"Job initialize failed.");
Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}"); Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}");
return await CompleteJobAsync(server, jobContext, message, TaskResult.Failed); return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed);
} }
// trace out all steps // trace out all steps
@@ -194,7 +182,7 @@ namespace GitHub.Runner.Worker
// Log the error and fail the job. // Log the error and fail the job.
Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}"); Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}");
jobContext.Error(ex); jobContext.Error(ex);
return await CompleteJobAsync(server, jobContext, message, TaskResult.Failed); return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed);
} }
finally finally
{ {
@@ -205,7 +193,7 @@ namespace GitHub.Runner.Worker
Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}"); Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}");
Trace.Info("Completing the job execution context."); Trace.Info("Completing the job execution context.");
return await CompleteJobAsync(server, jobContext, message); return await CompleteJobAsync(jobServer, jobContext, message);
} }
finally finally
{ {
@@ -219,66 +207,6 @@ namespace GitHub.Runner.Worker
} }
} }
private async Task<TaskResult> CompleteJobAsync(IRunnerService server, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult? taskResult = null)
{
if (server is IRunServer runServer)
{
return await CompleteJobAsync(runServer, jobContext, message, taskResult);
}
else if (server is IJobServer jobServer)
{
return await CompleteJobAsync(jobServer, jobContext, message, taskResult);
}
else
{
throw new NotSupportedException();
}
}
private async Task<TaskResult> CompleteJobAsync(IRunServer runServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult? taskResult = null)
{
jobContext.Debug($"Finishing: {message.JobDisplayName}");
TaskResult result = jobContext.Complete(taskResult);
if (jobContext.Global.Variables.TryGetValue("Node12ActionsWarnings", out var node12Warnings))
{
var actions = string.Join(", ", StringUtil.ConvertFromJson<HashSet<string>>(node12Warnings));
jobContext.Warning(string.Format(Constants.Runner.Node12DetectedAfterEndOfLife, actions));
}
// Make sure to clean temp after file upload since they may be pending fileupload still use the TEMP dir.
_tempDirectoryManager?.CleanupTempDirectory();
// Load any upgrade telemetry
LoadFromTelemetryFile(jobContext.Global.JobTelemetry);
// Make sure we don't submit secrets as telemetry
MaskTelemetrySecrets(jobContext.Global.JobTelemetry);
Trace.Info($"Raising job completed against run service");
var completeJobRetryLimit = 5;
var exceptions = new List<Exception>();
while (completeJobRetryLimit-- > 0)
{
try
{
await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, default);
return result;
}
catch (Exception ex)
{
Trace.Error($"Catch exception while attempting to complete job {message.JobId}, job request {message.RequestId}.");
Trace.Error(ex);
exceptions.Add(ex);
}
// delay 5 seconds before next retry.
await Task.Delay(TimeSpan.FromSeconds(5));
}
// rethrow exceptions from all attempts.
throw new AggregateException(exceptions);
}
private async Task<TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult? taskResult = null) private async Task<TaskResult> CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult? taskResult = null)
{ {
jobContext.Debug($"Finishing: {message.JobDisplayName}"); jobContext.Debug($"Finishing: {message.JobDisplayName}");

View File

@@ -1,9 +1,12 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.Expressions2;
using GitHub.DistributedTask.ObjectTemplating.Tokens; using GitHub.DistributedTask.ObjectTemplating.Tokens;
using GitHub.DistributedTask.Pipelines;
using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.Pipelines.ObjectTemplating; using GitHub.DistributedTask.Pipelines.ObjectTemplating;
using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.WebApi;
@@ -11,6 +14,8 @@ using GitHub.Runner.Common;
using GitHub.Runner.Common.Util; using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;
using GitHub.Runner.Worker.Expressions; using GitHub.Runner.Worker.Expressions;
using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating;
using Pipelines = GitHub.DistributedTask.Pipelines;
namespace GitHub.Runner.Worker namespace GitHub.Runner.Worker
{ {
@@ -21,8 +26,6 @@ namespace GitHub.Runner.Worker
string DisplayName { get; set; } string DisplayName { get; set; }
IExecutionContext ExecutionContext { get; set; } IExecutionContext ExecutionContext { get; set; }
TemplateToken Timeout { get; } TemplateToken Timeout { get; }
bool TryUpdateDisplayName(out bool updated);
bool EvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context, out bool updated);
Task RunAsync(); Task RunAsync();
} }
@@ -192,12 +195,6 @@ namespace GitHub.Runner.Worker
} }
else else
{ {
// This is our last, best chance to expand the display name. (At this point, all the requirements for successful expansion should be met.)
// That being said, evaluating the display name should still be considered as a "best effort" exercise. (It's not critical or paramount.)
// For that reason, we call a safe "Try..." wrapper method to ensure that any potential problems we encounter in evaluating the display name
// don't interfere with our ultimate goal within this code block: evaluation of the condition.
step.TryUpdateDisplayName(out _);
try try
{ {
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionTraceWriter); var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionTraceWriter);
@@ -259,6 +256,14 @@ namespace GitHub.Runner.Worker
private async Task RunStepAsync(IStep step, CancellationToken jobCancellationToken) private async Task RunStepAsync(IStep step, CancellationToken jobCancellationToken)
{ {
// Check to see if we can expand the display name
if (step is IActionRunner actionRunner &&
actionRunner.Stage == ActionRunStage.Main &&
actionRunner.TryEvaluateDisplayName(step.ExecutionContext.ExpressionValues, step.ExecutionContext))
{
step.ExecutionContext.UpdateTimelineRecordDisplayName(actionRunner.DisplayName);
}
// Start the step // Start the step
Trace.Info("Starting the step."); Trace.Info("Starting the step.");
step.ExecutionContext.Debug($"Starting: {step.DisplayName}"); step.ExecutionContext.Debug($"Starting: {step.DisplayName}");

View File

@@ -0,0 +1,20 @@
using GitHub.Services.OAuth;
namespace GitHub.Services.Common
{
public static class VssCredentialsExtension
{
public static VssOAuthCredential ToOAuthCredentials(
this VssCredentials credentials)
{
if (credentials.Federated.CredentialType == VssCredentialsType.OAuth)
{
return credentials.Federated as VssOAuthCredential;
}
else
{
return null;
}
}
}
}

View File

@@ -12,20 +12,20 @@ namespace GitHub.Services.Common
public class RawHttpMessageHandler: HttpMessageHandler public class RawHttpMessageHandler: HttpMessageHandler
{ {
public RawHttpMessageHandler( public RawHttpMessageHandler(
FederatedCredential credentials) VssOAuthCredential credentials)
: this(credentials, new RawClientHttpRequestSettings()) : this(credentials, new RawClientHttpRequestSettings())
{ {
} }
public RawHttpMessageHandler( public RawHttpMessageHandler(
FederatedCredential credentials, VssOAuthCredential credentials,
RawClientHttpRequestSettings settings) RawClientHttpRequestSettings settings)
: this(credentials, settings, new HttpClientHandler()) : this(credentials, settings, new HttpClientHandler())
{ {
} }
public RawHttpMessageHandler( public RawHttpMessageHandler(
FederatedCredential credentials, VssOAuthCredential credentials,
RawClientHttpRequestSettings settings, RawClientHttpRequestSettings settings,
HttpMessageHandler innerHandler) HttpMessageHandler innerHandler)
{ {
@@ -56,7 +56,7 @@ namespace GitHub.Services.Common
/// <summary> /// <summary>
/// Gets the credentials associated with this handler. /// Gets the credentials associated with this handler.
/// </summary> /// </summary>
public FederatedCredential Credentials public VssOAuthCredential Credentials
{ {
get; get;
private set; private set;
@@ -111,7 +111,7 @@ namespace GitHub.Services.Common
// Ensure that we attempt to use the most appropriate authentication mechanism by default. // Ensure that we attempt to use the most appropriate authentication mechanism by default.
if (m_tokenProvider == null) if (m_tokenProvider == null)
{ {
m_tokenProvider = this.Credentials.CreateTokenProvider(request.RequestUri, null, null); m_tokenProvider = this.Credentials.GetTokenProvider(request.RequestUri);
} }
} }
@@ -254,7 +254,7 @@ namespace GitHub.Services.Common
private CredentialWrapper m_credentialWrapper; private CredentialWrapper m_credentialWrapper;
private object m_thisLock; private object m_thisLock;
private const Int32 m_maxAuthRetries = 3; private const Int32 m_maxAuthRetries = 3;
private IssuedTokenProvider m_tokenProvider; private VssOAuthTokenProvider m_tokenProvider;
//.Net Core does not attempt NTLM schema on Linux, unless ICredentials is a CredentialCache instance //.Net Core does not attempt NTLM schema on Linux, unless ICredentials is a CredentialCache instance
//This workaround may not be needed after this corefx fix is consumed: https://github.com/dotnet/corefx/pull/7923 //This workaround may not be needed after this corefx fix is consumed: https://github.com/dotnet/corefx/pull/7923

View File

@@ -27,6 +27,18 @@ namespace GitHub.Services.Common
} }
} }
public static string TrimExcess(string text, int maxLength)
{
string result = text;
if (!string.IsNullOrEmpty(result) && result.Length > maxLength)
{
result = result.Substring(0, maxLength);
}
return result;
}
public static string ToBase64StringNoPaddingFromString(string utf8String) public static string ToBase64StringNoPaddingFromString(string utf8String)
{ {
return ToBase64StringNoPadding(Encoding.UTF8.GetBytes(utf8String)); return ToBase64StringNoPadding(Encoding.UTF8.GetBytes(utf8String));

View File

@@ -42,10 +42,9 @@ namespace GitHub.DistributedTask.Pipelines
IList<String> fileTable, IList<String> fileTable,
TemplateToken jobOutputs, TemplateToken jobOutputs,
IList<TemplateToken> defaults, IList<TemplateToken> defaults,
ActionsEnvironmentReference actionsEnvironment, ActionsEnvironmentReference actionsEnvironment)
String messageType = JobRequestMessageTypes.PipelineAgentJobRequest)
{ {
this.MessageType = messageType; this.MessageType = JobRequestMessageTypes.PipelineAgentJobRequest;
this.Plan = plan; this.Plan = plan;
this.JobId = jobId; this.JobId = jobId;
this.JobDisplayName = jobDisplayName; this.JobDisplayName = jobDisplayName;

View File

@@ -1,4 +1,4 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.ComponentModel; using System.ComponentModel;
using System.Linq; using System.Linq;
@@ -164,14 +164,13 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
public Dictionary<String, String> EvaluateStepInputs( public Dictionary<String, String> EvaluateStepInputs(
TemplateToken token, TemplateToken token,
DictionaryContextData contextData, DictionaryContextData contextData,
IList<IFunctionInfo> expressionFunctions, IList<IFunctionInfo> expressionFunctions)
IEnumerable<KeyValuePair<String, Object>> expressionState = null)
{ {
var result = default(Dictionary<String, String>); var result = default(Dictionary<String, String>);
if (token != null && token.Type != TokenType.Null) if (token != null && token.Type != TokenType.Null)
{ {
var context = CreateContext(contextData, expressionFunctions, expressionState); var context = CreateContext(contextData, expressionFunctions);
try try
{ {
token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepWith, token, 0, null, omitHeader: true); token = TemplateEvaluator.Evaluate(context, PipelineTemplateConstants.StepWith, token, 0, null, omitHeader: true);
@@ -456,6 +455,7 @@ namespace GitHub.DistributedTask.Pipelines.ObjectTemplating
private readonly String[] s_expressionValueNames = new[] private readonly String[] s_expressionValueNames = new[]
{ {
PipelineTemplateConstants.GitHub, PipelineTemplateConstants.GitHub,
PipelineTemplateConstants.Needs,
PipelineTemplateConstants.Strategy, PipelineTemplateConstants.Strategy,
PipelineTemplateConstants.Matrix, PipelineTemplateConstants.Matrix,
PipelineTemplateConstants.Needs, PipelineTemplateConstants.Needs,

View File

@@ -1,30 +1,38 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Runtime.Serialization; using System.Runtime.Serialization;
using GitHub.Services.Common;
namespace GitHub.DistributedTask.WebApi namespace GitHub.DistributedTask.WebApi
{ {
public interface IReadOnlyIssue
{
IssueType Type { get; }
string Category { get; }
string Message { get; }
bool? IsInfrastructureIssue { get; }
string this[string key] { get; }
}
[DataContract] [DataContract]
public class Issue public class Issue : IReadOnlyIssue
{ {
public Issue() public Issue()
: this(null)
{ {
} }
private Issue(Issue issueToBeCloned) private Issue(Issue original)
{ {
this.Type = issueToBeCloned.Type; this.EnsureInitialized();
this.Category = issueToBeCloned.Category; if (original != null)
this.Message = issueToBeCloned.Message;
this.IsInfrastructureIssue = issueToBeCloned.IsInfrastructureIssue;
if (issueToBeCloned.m_data != null)
{ {
foreach (var item in issueToBeCloned.m_data) this.Type = original.Type;
{ this.Category = original.Category;
this.Data.Add(item); this.Message = original.Message;
} this.IsInfrastructureIssue = original.IsInfrastructureIssue;
m_data.AddRange(original.m_data);
} }
} }
@@ -36,14 +44,14 @@ namespace GitHub.DistributedTask.WebApi
} }
[DataMember(Order = 2)] [DataMember(Order = 2)]
public String Category public string Category
{ {
get; get;
set; set;
} }
[DataMember(Order = 3)] [DataMember(Order = 3)]
public String Message public string Message
{ {
get; get;
set; set;
@@ -56,15 +64,16 @@ namespace GitHub.DistributedTask.WebApi
set; set;
} }
public IDictionary<String, String> Data public string this[string key]
{ {
get get
{ {
if (m_data == null) m_data.TryGetValue(key, out string result);
{ return result;
m_data = new Dictionary<String, String>(StringComparer.OrdinalIgnoreCase);
} }
return m_data; set
{
m_data[key] = value;
} }
} }
@@ -77,6 +86,7 @@ namespace GitHub.DistributedTask.WebApi
private void OnDeserialized(StreamingContext context) private void OnDeserialized(StreamingContext context)
{ {
SerializationHelper.Copy(ref m_serializedData, ref m_data, StringComparer.OrdinalIgnoreCase, true); SerializationHelper.Copy(ref m_serializedData, ref m_data, StringComparer.OrdinalIgnoreCase, true);
this.EnsureInitialized();
} }
[OnSerializing] [OnSerializing]
@@ -91,9 +101,21 @@ namespace GitHub.DistributedTask.WebApi
m_serializedData = null; m_serializedData = null;
} }
[DataMember(Name = "Data", EmitDefaultValue = false, Order = 4)] /// <summary>
private IDictionary<String, String> m_serializedData; /// DataContractSerializer bypasses all constructor logic and inline initialization!
/// This method takes the place of a workhorse constructor for baseline initialization.
/// The expectation is for this logic to be accessible to constructors and also to the OnDeserialized helper.
/// </summary>
private void EnsureInitialized()
{
//Note that ?? is a short-circuiting operator.
m_data = m_data ?? new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
}
[DataMember(Name = "Data", EmitDefaultValue = false, Order = 4)]
private IDictionary<string, string> m_serializedData;
private IDictionary<string, string> m_data;
private IDictionary<String, String> m_data;
} }
} }

View File

@@ -0,0 +1,75 @@
using System;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Services.Common;
using GitHub.Services.OAuth;
using GitHub.Services.WebApi;
using Sdk.WebApi.WebApi;
namespace GitHub.DistributedTask.WebApi
{
[ResourceArea(TaskResourceIds.AreaId)]
public class RunServiceHttpClient : RawHttpClientBase
{
public RunServiceHttpClient(
Uri baseUrl,
VssOAuthCredential credentials)
: base(baseUrl, credentials)
{
}
public RunServiceHttpClient(
Uri baseUrl,
VssOAuthCredential credentials,
RawClientHttpRequestSettings settings)
: base(baseUrl, credentials, settings)
{
}
public RunServiceHttpClient(
Uri baseUrl,
VssOAuthCredential credentials,
params DelegatingHandler[] handlers)
: base(baseUrl, credentials, handlers)
{
}
public RunServiceHttpClient(
Uri baseUrl,
VssOAuthCredential credentials,
RawClientHttpRequestSettings settings,
params DelegatingHandler[] handlers)
: base(baseUrl, credentials, settings, handlers)
{
}
public RunServiceHttpClient(
Uri baseUrl,
HttpMessageHandler pipeline,
Boolean disposeHandler)
: base(baseUrl, pipeline, disposeHandler)
{
}
public Task<Pipelines.AgentJobRequestMessage> GetJobMessageAsync(
Uri requestUri,
string messageId,
CancellationToken cancellationToken = default)
{
HttpMethod httpMethod = new HttpMethod("POST");
var payload = new {
StreamID = messageId
};
var payloadJson = JsonUtility.ToString(payload);
var requestContent = new StringContent(payloadJson, System.Text.Encoding.UTF8, "application/json");
return SendAsync<Pipelines.AgentJobRequestMessage>(
httpMethod,
additionalHeaders: null,
requestUri: requestUri,
content: requestContent,
cancellationToken: cancellationToken);
}
}
}

View File

@@ -1,4 +1,4 @@
using GitHub.Services.Common; using GitHub.Services.Common;
using GitHub.Services.WebApi; using GitHub.Services.WebApi;
using System; using System;
using System.Runtime.Serialization; using System.Runtime.Serialization;
@@ -28,6 +28,7 @@ namespace GitHub.DistributedTask.WebApi
this.Name = name; this.Name = name;
} }
[DataMember] [DataMember]
public String Type public String Type
{ {
@@ -100,7 +101,6 @@ namespace GitHub.DistributedTask.WebApi
public static readonly String Summary = "DistributedTask.Core.Summary"; public static readonly String Summary = "DistributedTask.Core.Summary";
public static readonly String FileAttachment = "DistributedTask.Core.FileAttachment"; public static readonly String FileAttachment = "DistributedTask.Core.FileAttachment";
public static readonly String DiagnosticLog = "DistributedTask.Core.DiagnosticLog"; public static readonly String DiagnosticLog = "DistributedTask.Core.DiagnosticLog";
public static readonly String ResultsLog = "Results.Core.Log";
} }
[GenerateAllConstants] [GenerateAllConstants]

View File

@@ -10,11 +10,14 @@ namespace GitHub.DistributedTask.WebApi
public sealed class TimelineRecord public sealed class TimelineRecord
{ {
public TimelineRecord() public TimelineRecord()
: this(null)
{ {
this.Attempt = 1;
} }
private TimelineRecord(TimelineRecord recordToBeCloned) private TimelineRecord(TimelineRecord recordToBeCloned)
{
this.EnsureInitialized();
if (recordToBeCloned != null)
{ {
this.Attempt = recordToBeCloned.Attempt; this.Attempt = recordToBeCloned.Attempt;
this.ChangeId = recordToBeCloned.ChangeId; this.ChangeId = recordToBeCloned.ChangeId;
@@ -60,19 +63,23 @@ namespace GitHub.DistributedTask.WebApi
}; };
} }
if (recordToBeCloned.m_issues?.Count> 0) if (recordToBeCloned.m_issues?.Count > 0)
{ {
this.Issues.AddRange(recordToBeCloned.Issues.Select(i => i.Clone())); m_issues.AddRange(recordToBeCloned.m_issues.Select(i => i.Clone()));
} }
if (recordToBeCloned.m_previousAttempts?.Count > 0) if (recordToBeCloned.m_previousAttempts?.Count > 0)
{ {
this.PreviousAttempts.AddRange(recordToBeCloned.PreviousAttempts); m_previousAttempts.AddRange(recordToBeCloned.m_previousAttempts);
} }
if (recordToBeCloned.m_variables?.Count > 0) if (recordToBeCloned.m_variables?.Count > 0)
{ {
this.m_variables = recordToBeCloned.Variables.ToDictionary(k => k.Key, v => v.Value.Clone()); // Don't pave over the case-insensitive Dictionary we initialized above.
foreach (var kvp in recordToBeCloned.m_variables) {
m_variables[kvp.Key] = kvp.Value.Clone();
}
}
} }
} }
@@ -234,10 +241,6 @@ namespace GitHub.DistributedTask.WebApi
{ {
get get
{ {
if (m_issues == null)
{
m_issues = new List<Issue>();
}
return m_issues; return m_issues;
} }
} }
@@ -274,22 +277,14 @@ namespace GitHub.DistributedTask.WebApi
{ {
get get
{ {
if (m_previousAttempts == null)
{
m_previousAttempts = new List<TimelineAttempt>();
}
return m_previousAttempts; return m_previousAttempts;
} }
} }
public IDictionary<String, VariableValue> Variables public IDictionary<string, VariableValue> Variables
{ {
get get
{ {
if (m_variables == null)
{
m_variables = new Dictionary<String, VariableValue>(StringComparer.OrdinalIgnoreCase);
}
return m_variables; return m_variables;
} }
} }
@@ -299,11 +294,32 @@ namespace GitHub.DistributedTask.WebApi
return new TimelineRecord(this); return new TimelineRecord(this);
} }
[OnDeserialized]
private void OnDeserialized(StreamingContext context)
{
this.EnsureInitialized();
}
/// <summary>
/// DataContractSerializer bypasses all constructor logic and inline initialization!
/// This method takes the place of a workhorse constructor for baseline initialization.
/// The expectation is for this logic to be accessible to constructors and also to the OnDeserialized helper.
/// </summary>
private void EnsureInitialized()
{
//Note that ?? is a short-circuiting operator.
m_issues = m_issues ?? new List<Issue>();
m_variables = m_variables ?? new Dictionary<string, VariableValue>(StringComparer.OrdinalIgnoreCase);
m_previousAttempts = m_previousAttempts ?? new List<TimelineAttempt>();
this.Attempt = Math.Max(this.Attempt, 1);
}
[DataMember(Name = "Issues", EmitDefaultValue = false, Order = 60)] [DataMember(Name = "Issues", EmitDefaultValue = false, Order = 60)]
private List<Issue> m_issues; private List<Issue> m_issues;
[DataMember(Name = "Variables", EmitDefaultValue = false, Order = 80)] [DataMember(Name = "Variables", EmitDefaultValue = false, Order = 80)]
private Dictionary<String, VariableValue> m_variables; private Dictionary<string, VariableValue> m_variables;
[DataMember(Name = "PreviousAttempts", EmitDefaultValue = false, Order = 120)] [DataMember(Name = "PreviousAttempts", EmitDefaultValue = false, Order = 120)]
private List<TimelineAttempt> m_previousAttempts; private List<TimelineAttempt> m_previousAttempts;

View File

@@ -1,13 +0,0 @@
using System.Collections.Generic;
using System.Runtime.Serialization;
using GitHub.DistributedTask.WebApi;
namespace GitHub.Actions.RunService.WebApi
{
[DataContract]
public class AcquireJobRequest
{
[DataMember(Name = "streamId", EmitDefaultValue = false)]
public string StreamID { get; set; }
}
}

View File

@@ -1,26 +0,0 @@
using System;
using System.Collections.Generic;
using System.Runtime.Serialization;
using GitHub.DistributedTask.WebApi;
namespace GitHub.Actions.RunService.WebApi
{
[DataContract]
public class CompleteJobRequest
{
[DataMember(Name = "planId", EmitDefaultValue = false)]
public Guid PlanID { get; set; }
[DataMember(Name = "jobId", EmitDefaultValue = false)]
public Guid JobID { get; set; }
[DataMember(Name = "conclusion")]
public TaskResult Conclusion { get; set; }
[DataMember(Name = "outputs", EmitDefaultValue = false)]
public Dictionary<string, VariableValue> Outputs { get; set; }
[DataMember(Name = "stepResults", EmitDefaultValue = false)]
public IList<StepResult> StepResults { get; set; }
}
}

View File

@@ -1,15 +0,0 @@
using System;
using System.Runtime.Serialization;
namespace GitHub.Actions.RunService.WebApi
{
[DataContract]
public class RenewJobRequest
{
[DataMember(Name = "planId", EmitDefaultValue = false)]
public Guid PlanID { get; set; }
[DataMember(Name = "jobId", EmitDefaultValue = false)]
public Guid JobID { get; set; }
}
}

View File

@@ -1,16 +0,0 @@
using System;
using System.Runtime.Serialization;
namespace Sdk.RSWebApi.Contracts
{
[DataContract]
public class RenewJobResponse
{
[DataMember]
public DateTime LockedUntil
{
get;
internal set;
}
}
}

View File

@@ -1,38 +0,0 @@
using System;
using System.Runtime.Serialization;
using System.Threading.Tasks;
using GitHub.DistributedTask.WebApi;
namespace GitHub.Actions.RunService.WebApi
{
[DataContract]
public class StepResult
{
[DataMember(Name = "external_id", EmitDefaultValue = false)]
public Guid ExternalID { get; set; }
[DataMember(Name = "number", EmitDefaultValue = false)]
public int? Number { get; set; }
[DataMember(Name = "name", EmitDefaultValue = false)]
public string Name { get; set; }
[DataMember(Name = "status")]
public TimelineRecordState? Status { get; set; }
[DataMember(Name = "conclusion")]
public TaskResult? Conclusion { get; set; }
[DataMember(Name = "started_at", EmitDefaultValue = false)]
public DateTime? StartedAt { get; set; }
[DataMember(Name = "completed_at", EmitDefaultValue = false)]
public DateTime? CompletedAt { get; set; }
[DataMember(Name = "completed_log_url", EmitDefaultValue = false)]
public string CompletedLogURL { get; set; }
[DataMember(Name = "completed_log_lines", EmitDefaultValue = false)]
public long? CompletedLogLines { get; set; }
}
}

View File

@@ -1,131 +0,0 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using GitHub.DistributedTask.Pipelines;
using GitHub.DistributedTask.WebApi;
using GitHub.Services.Common;
using GitHub.Services.OAuth;
using GitHub.Services.WebApi;
using Sdk.RSWebApi.Contracts;
using Sdk.WebApi.WebApi;
namespace GitHub.Actions.RunService.WebApi
{
public class RunServiceHttpClient : RawHttpClientBase
{
public RunServiceHttpClient(
Uri baseUrl,
VssOAuthCredential credentials)
: base(baseUrl, credentials)
{
}
public RunServiceHttpClient(
Uri baseUrl,
VssOAuthCredential credentials,
RawClientHttpRequestSettings settings)
: base(baseUrl, credentials, settings)
{
}
public RunServiceHttpClient(
Uri baseUrl,
VssOAuthCredential credentials,
params DelegatingHandler[] handlers)
: base(baseUrl, credentials, handlers)
{
}
public RunServiceHttpClient(
Uri baseUrl,
VssOAuthCredential credentials,
RawClientHttpRequestSettings settings,
params DelegatingHandler[] handlers)
: base(baseUrl, credentials, settings, handlers)
{
}
public RunServiceHttpClient(
Uri baseUrl,
HttpMessageHandler pipeline,
Boolean disposeHandler)
: base(baseUrl, pipeline, disposeHandler)
{
}
public Task<AgentJobRequestMessage> GetJobMessageAsync(
Uri requestUri,
string messageId,
CancellationToken cancellationToken = default)
{
HttpMethod httpMethod = new HttpMethod("POST");
var payload = new AcquireJobRequest
{
StreamID = messageId
};
requestUri = new Uri(requestUri, "acquirejob");
var requestContent = new ObjectContent<AcquireJobRequest>(payload, new VssJsonMediaTypeFormatter(true));
return SendAsync<AgentJobRequestMessage>(
httpMethod,
requestUri: requestUri,
content: requestContent,
cancellationToken: cancellationToken);
}
public Task CompleteJobAsync(
Uri requestUri,
Guid planId,
Guid jobId,
TaskResult result,
Dictionary<String, VariableValue> outputs,
IList<StepResult> stepResults,
CancellationToken cancellationToken = default)
{
HttpMethod httpMethod = new HttpMethod("POST");
var payload = new CompleteJobRequest()
{
PlanID = planId,
JobID = jobId,
Conclusion = result,
Outputs = outputs,
StepResults = stepResults
};
requestUri = new Uri(requestUri, "completejob");
var requestContent = new ObjectContent<CompleteJobRequest>(payload, new VssJsonMediaTypeFormatter(true));
return SendAsync(
httpMethod,
requestUri,
content: requestContent,
cancellationToken: cancellationToken);
}
public Task<RenewJobResponse> RenewJobAsync(
Uri requestUri,
Guid planId,
Guid jobId,
CancellationToken cancellationToken = default)
{
HttpMethod httpMethod = new HttpMethod("POST");
var payload = new RenewJobRequest()
{
PlanID = planId,
JobID = jobId
};
requestUri = new Uri(requestUri, "renewjob");
var requestContent = new ObjectContent<RenewJobRequest>(payload, new VssJsonMediaTypeFormatter(true));
return SendAsync<RenewJobResponse>(
httpMethod,
requestUri,
content: requestContent,
cancellationToken: cancellationToken);
}
}
}

View File

@@ -1,134 +0,0 @@
using System.Runtime.Serialization;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
namespace GitHub.Services.Results.Contracts
{
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class GetSignedStepSummaryURLRequest
{
[DataMember]
public string WorkflowJobRunBackendId;
[DataMember]
public string WorkflowRunBackendId;
[DataMember]
public string StepBackendId;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class GetSignedStepSummaryURLResponse
{
[DataMember]
public string SummaryUrl;
[DataMember]
public long SoftSizeLimit;
[DataMember]
public string BlobStorageType;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class StepSummaryMetadataCreate
{
[DataMember]
public string StepBackendId;
[DataMember]
public string WorkflowRunBackendId;
[DataMember]
public string WorkflowJobRunBackendId;
[DataMember]
public long Size;
[DataMember]
public string UploadedAt;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class GetSignedJobLogsURLRequest
{
[DataMember]
public string WorkflowJobRunBackendId;
[DataMember]
public string WorkflowRunBackendId;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class GetSignedJobLogsURLResponse
{
[DataMember]
public string LogsUrl;
[DataMember]
public string BlobStorageType;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class GetSignedStepLogsURLRequest
{
[DataMember]
public string WorkflowJobRunBackendId;
[DataMember]
public string WorkflowRunBackendId;
[DataMember]
public string StepBackendId;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class GetSignedStepLogsURLResponse
{
[DataMember]
public string LogsUrl;
[DataMember]
public string BlobStorageType;
[DataMember]
public long SoftSizeLimit;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class JobLogsMetadataCreate
{
[DataMember]
public string WorkflowRunBackendId;
[DataMember]
public string WorkflowJobRunBackendId;
[DataMember]
public string UploadedAt;
[DataMember]
public long LineCount;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class StepLogsMetadataCreate
{
[DataMember]
public string WorkflowRunBackendId;
[DataMember]
public string WorkflowJobRunBackendId;
[DataMember]
public string StepBackendId;
[DataMember]
public string UploadedAt;
[DataMember]
public long LineCount;
}
[DataContract]
[JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))]
public class CreateMetadataResponse
{
[DataMember]
public bool Ok;
}
public static class BlobStorageTypes
{
public static readonly string AzureBlobStorage = "BLOB_STORAGE_TYPE_AZURE";
public static readonly string Unspecified = "BLOB_STORAGE_TYPE_UNSPECIFIED";
}
}

View File

@@ -101,17 +101,6 @@ namespace Sdk.WebApi.WebApi
} }
} }
protected Task<T> SendAsync<T>(
HttpMethod method,
Uri requestUri,
HttpContent content = null,
IEnumerable<KeyValuePair<String, String>> queryParameters = null,
Object userState = null,
CancellationToken cancellationToken = default(CancellationToken))
{
return SendAsync<T>(method, null, requestUri, content, queryParameters, userState, cancellationToken);
}
protected async Task<T> SendAsync<T>( protected async Task<T> SendAsync<T>(
HttpMethod method, HttpMethod method,
IEnumerable<KeyValuePair<String, String>> additionalHeaders, IEnumerable<KeyValuePair<String, String>> additionalHeaders,

View File

@@ -1,341 +0,0 @@
using System;
using System.IO;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Services.Results.Contracts;
using System.Net.Http.Formatting;
using Sdk.WebApi.WebApi;
namespace GitHub.Services.Results.Client
{
public class ResultsHttpClient : RawHttpClientBase
{
public ResultsHttpClient(
Uri baseUrl,
HttpMessageHandler pipeline,
string token,
bool disposeHandler)
: base(baseUrl, pipeline, disposeHandler)
{
m_token = token;
m_resultsServiceUrl = baseUrl;
m_formatter = new JsonMediaTypeFormatter();
}
// Get Sas URL calls
private async Task<T> GetResultsSignedURLResponse<R, T>(Uri uri, CancellationToken cancellationToken, R request)
{
using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, uri))
{
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token);
requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json"));
using (HttpContent content = new ObjectContent<R>(request, m_formatter))
{
requestMessage.Content = content;
using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken))
{
return await ReadJsonContentAsync<T>(response, cancellationToken);
}
}
}
}
private async Task<GetSignedStepSummaryURLResponse> GetStepSummaryUploadUrlAsync(string planId, string jobId, Guid stepId, CancellationToken cancellationToken)
{
var request = new GetSignedStepSummaryURLRequest()
{
WorkflowJobRunBackendId = jobId,
WorkflowRunBackendId = planId,
StepBackendId = stepId.ToString()
};
var getStepSummarySignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetStepSummarySignedBlobURL);
return await GetResultsSignedURLResponse<GetSignedStepSummaryURLRequest, GetSignedStepSummaryURLResponse>(getStepSummarySignedBlobURLEndpoint, cancellationToken, request);
}
private async Task<GetSignedStepLogsURLResponse> GetStepLogUploadUrlAsync(string planId, string jobId, Guid stepId, CancellationToken cancellationToken)
{
var request = new GetSignedStepLogsURLRequest()
{
WorkflowJobRunBackendId = jobId,
WorkflowRunBackendId = planId,
StepBackendId = stepId.ToString(),
};
var getStepLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetStepLogsSignedBlobURL);
return await GetResultsSignedURLResponse<GetSignedStepLogsURLRequest, GetSignedStepLogsURLResponse>(getStepLogsSignedBlobURLEndpoint, cancellationToken, request);
}
private async Task<GetSignedJobLogsURLResponse> GetJobLogUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken)
{
var request = new GetSignedJobLogsURLRequest()
{
WorkflowJobRunBackendId = jobId,
WorkflowRunBackendId = planId,
};
var getJobLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetJobLogsSignedBlobURL);
return await GetResultsSignedURLResponse<GetSignedJobLogsURLRequest, GetSignedJobLogsURLResponse>(getJobLogsSignedBlobURLEndpoint, cancellationToken, request);
}
// Create metadata calls
private async Task CreateMetadata<R>(Uri uri, CancellationToken cancellationToken, R request, string timestamp)
{
using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, uri))
{
requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token);
requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json"));
using (HttpContent content = new ObjectContent<R>(request, m_formatter))
{
requestMessage.Content = content;
using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken))
{
var jsonResponse = await ReadJsonContentAsync<CreateMetadataResponse>(response, cancellationToken);
if (!jsonResponse.Ok)
{
throw new Exception($"Failed to mark {typeof(R).Name} upload as complete, status code: {response.StatusCode}, ok: {jsonResponse.Ok}, timestamp: {timestamp}");
}
}
}
}
}
private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, Guid stepId, long size, CancellationToken cancellationToken)
{
var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat);
var request = new StepSummaryMetadataCreate()
{
WorkflowJobRunBackendId = jobId,
WorkflowRunBackendId = planId,
StepBackendId = stepId.ToString(),
Size = size,
UploadedAt = timestamp
};
var createStepSummaryMetadataEndpoint = new Uri(m_resultsServiceUrl, Constants.CreateStepSummaryMetadata);
await CreateMetadata<StepSummaryMetadataCreate>(createStepSummaryMetadataEndpoint, cancellationToken, request, timestamp);
}
private async Task StepLogUploadCompleteAsync(string planId, string jobId, Guid stepId, long lineCount, CancellationToken cancellationToken)
{
var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat);
var request = new StepLogsMetadataCreate()
{
WorkflowJobRunBackendId = jobId,
WorkflowRunBackendId = planId,
StepBackendId = stepId.ToString(),
UploadedAt = timestamp,
LineCount = lineCount,
};
var createStepLogsMetadataEndpoint = new Uri(m_resultsServiceUrl, Constants.CreateStepLogsMetadata);
await CreateMetadata<StepLogsMetadataCreate>(createStepLogsMetadataEndpoint, cancellationToken, request, timestamp);
}
private async Task JobLogUploadCompleteAsync(string planId, string jobId, long lineCount, CancellationToken cancellationToken)
{
var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat);
var request = new JobLogsMetadataCreate()
{
WorkflowJobRunBackendId = jobId,
WorkflowRunBackendId = planId,
UploadedAt = timestamp,
LineCount = lineCount,
};
var createJobLogsMetadataEndpoint = new Uri(m_resultsServiceUrl, Constants.CreateJobLogsMetadata);
await CreateMetadata<JobLogsMetadataCreate>(createJobLogsMetadataEndpoint, cancellationToken, request, timestamp);
}
private async Task<HttpResponseMessage> UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken)
{
// Upload the file to the url
var request = new HttpRequestMessage(HttpMethod.Put, url)
{
Content = new StreamContent(file)
};
if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob);
}
using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
{
if (!response.IsSuccessStatusCode)
{
throw new Exception($"Failed to upload file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}");
}
return response;
}
}
private async Task<HttpResponseMessage> CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken)
{
var request = new HttpRequestMessage(HttpMethod.Put, url)
{
Content = new StringContent("")
};
if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob);
request.Content.Headers.Add("Content-Length", "0");
}
using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
{
if (!response.IsSuccessStatusCode)
{
throw new Exception($"Failed to create append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}");
}
return response;
}
}
private async Task<HttpResponseMessage> UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken)
{
var comp = finalize ? "&comp=appendblock&seal=true" : "&comp=appendblock";
// Upload the file to the url
var request = new HttpRequestMessage(HttpMethod.Put, url + comp)
{
Content = new StreamContent(file)
};
if (blobStorageType == BlobStorageTypes.AzureBlobStorage)
{
request.Content.Headers.Add("Content-Length", fileSize.ToString());
request.Content.Headers.Add(Constants.AzureBlobSealedHeader, finalize.ToString());
}
using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken))
{
if (!response.IsSuccessStatusCode)
{
throw new Exception($"Failed to upload append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}, object: {response}, fileSize: {fileSize}");
}
return response;
}
}
// Handle file upload for step summary
public async Task UploadStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken)
{
// Get the upload url
var uploadUrlResponse = await GetStepSummaryUploadUrlAsync(planId, jobId, stepId, cancellationToken);
if (uploadUrlResponse == null)
{
throw new Exception("Failed to get step summary upload url");
}
// Do we want to throw an exception here or should we just be uploading/truncating the data
var fileSize = new FileInfo(file).Length;
if (fileSize > uploadUrlResponse.SoftSizeLimit)
{
throw new Exception($"File size is larger than the upload url allows, file size: {fileSize}, upload url size: {uploadUrlResponse.SoftSizeLimit}");
}
// Upload the file
using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
{
var response = await UploadBlockFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken);
}
// Send step summary upload complete message
await StepSummaryUploadCompleteAsync(planId, jobId, stepId, fileSize, cancellationToken);
}
// Handle file upload for step log
public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken)
{
// Get the upload url
var uploadUrlResponse = await GetStepLogUploadUrlAsync(planId, jobId, stepId, cancellationToken);
if (uploadUrlResponse == null || uploadUrlResponse.LogsUrl == null)
{
throw new Exception("Failed to get step log upload url");
}
// Create the Append blob
if (firstBlock)
{
await CreateAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken);
}
// Upload content
var fileSize = new FileInfo(file).Length;
using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
{
var response = await UploadAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, fileStream, finalize, fileSize, cancellationToken);
}
// Update metadata
if (finalize)
{
// Send step log upload complete message
await StepLogUploadCompleteAsync(planId, jobId, stepId, lineCount, cancellationToken);
}
}
// Handle file upload for job log
public async Task UploadResultsJobLogAsync(string planId, string jobId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken)
{
// Get the upload url
var uploadUrlResponse = await GetJobLogUploadUrlAsync(planId, jobId, cancellationToken);
if (uploadUrlResponse == null || uploadUrlResponse.LogsUrl == null)
{
throw new Exception("Failed to get job log upload url");
}
// Create the Append blob
if (firstBlock)
{
await CreateAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken);
}
// Upload content
var fileSize = new FileInfo(file).Length;
using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true))
{
var response = await UploadAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, fileStream, finalize, fileSize, cancellationToken);
}
// Update metadata
if (finalize)
{
// Send step log upload complete message
await JobLogUploadCompleteAsync(planId, jobId, lineCount, cancellationToken);
}
}
private MediaTypeFormatter m_formatter;
private Uri m_resultsServiceUrl;
private string m_token;
}
// Constants specific to results
public static class Constants
{
public static readonly string TimestampFormat = "yyyy-MM-dd'T'HH:mm:ss.fffK";
public static readonly string ResultsReceiverTwirpEndpoint = "twirp/results.services.receiver.Receiver/";
public static readonly string GetStepSummarySignedBlobURL = ResultsReceiverTwirpEndpoint + "GetStepSummarySignedBlobURL";
public static readonly string CreateStepSummaryMetadata = ResultsReceiverTwirpEndpoint + "CreateStepSummaryMetadata";
public static readonly string GetStepLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetStepLogsSignedBlobURL";
public static readonly string CreateStepLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateStepLogsMetadata";
public static readonly string GetJobLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetJobLogsSignedBlobURL";
public static readonly string CreateJobLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateJobLogsMetadata";
public static readonly string AzureBlobSealedHeader = "x-ms-blob-sealed";
public static readonly string AzureBlobTypeHeader = "x-ms-blob-type";
public static readonly string AzureBlockBlob = "BlockBlob";
public static readonly string AzureAppendBlob = "AppendBlob";
}
}

View File

@@ -824,7 +824,6 @@ namespace GitHub.Runner.Common.Tests
[InlineData("remove", "version")] [InlineData("remove", "version")]
[InlineData("remove", "commit")] [InlineData("remove", "commit")]
[InlineData("remove", "check")] [InlineData("remove", "check")]
[InlineData("remove", "local")]
[InlineData("run", "help")] [InlineData("run", "help")]
[InlineData("run", "version")] [InlineData("run", "version")]
[InlineData("run", "commit")] [InlineData("run", "commit")]

View File

@@ -1,17 +1,12 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Reflection; using System.Reflection;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
using GitHub.DistributedTask.Pipelines;
using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Listener; using GitHub.Runner.Listener;
using GitHub.Runner.Listener.Configuration;
using GitHub.Services.WebApi; using GitHub.Services.WebApi;
using Moq; using Moq;
using Sdk.RSWebApi.Contracts;
using Xunit; using Xunit;
using Pipelines = GitHub.DistributedTask.Pipelines; using Pipelines = GitHub.DistributedTask.Pipelines;
@@ -23,8 +18,6 @@ namespace GitHub.Runner.Common.Tests.Listener
private Mock<IProcessChannel> _processChannel; private Mock<IProcessChannel> _processChannel;
private Mock<IProcessInvoker> _processInvoker; private Mock<IProcessInvoker> _processInvoker;
private Mock<IRunnerServer> _runnerServer; private Mock<IRunnerServer> _runnerServer;
private Mock<IRunServer> _runServer;
private Mock<IConfigurationStore> _configurationStore; private Mock<IConfigurationStore> _configurationStore;
public JobDispatcherL0() public JobDispatcherL0()
@@ -32,7 +25,6 @@ namespace GitHub.Runner.Common.Tests.Listener
_processChannel = new Mock<IProcessChannel>(); _processChannel = new Mock<IProcessChannel>();
_processInvoker = new Mock<IProcessInvoker>(); _processInvoker = new Mock<IProcessInvoker>();
_runnerServer = new Mock<IRunnerServer>(); _runnerServer = new Mock<IRunnerServer>();
_runServer = new Mock<IRunServer>();
_configurationStore = new Mock<IConfigurationStore>(); _configurationStore = new Mock<IConfigurationStore>();
} }
@@ -147,7 +139,7 @@ namespace GitHub.Runner.Common.Tests.Listener
var jobDispatcher = new JobDispatcher(); var jobDispatcher = new JobDispatcher();
jobDispatcher.Initialize(hc); jobDispatcher.Initialize(hc);
await jobDispatcher.RenewJobRequestAsync(It.IsAny<AgentJobRequestMessage>(), It.IsAny<ServiceEndpoint>(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token);
Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully);
_runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny<int>(), It.IsAny<long>(), It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Exactly(5)); _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny<int>(), It.IsAny<long>(), It.IsAny<Guid>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Exactly(5));
@@ -205,7 +197,7 @@ namespace GitHub.Runner.Common.Tests.Listener
var jobDispatcher = new JobDispatcher(); var jobDispatcher = new JobDispatcher();
jobDispatcher.Initialize(hc); jobDispatcher.Initialize(hc);
await jobDispatcher.RenewJobRequestAsync(It.IsAny<AgentJobRequestMessage>(), It.IsAny<ServiceEndpoint>(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token);
Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed.");
Assert.False(cancellationTokenSource.IsCancellationRequested); Assert.False(cancellationTokenSource.IsCancellationRequested);
@@ -213,75 +205,6 @@ namespace GitHub.Runner.Common.Tests.Listener
} }
} }
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Runner")]
public async void DispatcherRenewJobOnRunServiceStopOnJobNotFoundExceptions()
{
//Arrange
using (var hc = new TestHostContext(this))
{
int poolId = 1;
Int64 requestId = 1000;
int count = 0;
var trace = hc.GetTrace(nameof(DispatcherRenewJobOnRunServiceStopOnJobNotFoundExceptions));
TaskCompletionSource<int> firstJobRequestRenewed = new();
CancellationTokenSource cancellationTokenSource = new();
TaskAgentJobRequest request = new();
PropertyInfo lockUntilProperty = request.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
Assert.NotNull(lockUntilProperty);
lockUntilProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5));
hc.SetSingleton<IRunServer>(_runServer.Object);
hc.SetSingleton<IConfigurationStore>(_configurationStore.Object);
_configurationStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings() { PoolId = 1 });
_ = _runServer.Setup(x => x.RenewJobAsync(It.IsAny<Guid>(), It.IsAny<Guid>(), It.IsAny<CancellationToken>()))
.Returns(() =>
{
count++;
if (!firstJobRequestRenewed.Task.IsCompletedSuccessfully)
{
trace.Info("First renew happens.");
}
if (count < 5)
{
var response = new RenewJobResponse()
{
LockedUntil = request.LockedUntil.Value
};
return Task.FromResult<RenewJobResponse>(response);
}
else if (count == 5)
{
cancellationTokenSource.CancelAfter(10000);
throw new TaskOrchestrationJobNotFoundException("");
}
else
{
throw new InvalidOperationException("Should not reach here.");
}
});
var jobDispatcher = new JobDispatcher();
jobDispatcher.Initialize(hc);
EnableRunServiceJobForJobDispatcher(jobDispatcher);
// Set the value of the _isRunServiceJob field to true
var isRunServiceJobField = typeof(JobDispatcher).GetField("_isRunServiceJob", BindingFlags.NonPublic | BindingFlags.Instance);
isRunServiceJobField.SetValue(jobDispatcher, true);
await jobDispatcher.RenewJobRequestAsync(GetAgentJobRequestMessage(), GetServiceEndpoint(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token);
Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed.");
Assert.False(cancellationTokenSource.IsCancellationRequested);
_runServer.Verify(x => x.RenewJobAsync(It.IsAny<Guid>(), It.IsAny<Guid>(), It.IsAny<CancellationToken>()), Times.Exactly(5));
}
}
[Fact] [Fact]
[Trait("Level", "L0")] [Trait("Level", "L0")]
[Trait("Category", "Runner")] [Trait("Category", "Runner")]
@@ -333,7 +256,7 @@ namespace GitHub.Runner.Common.Tests.Listener
var jobDispatcher = new JobDispatcher(); var jobDispatcher = new JobDispatcher();
jobDispatcher.Initialize(hc); jobDispatcher.Initialize(hc);
await jobDispatcher.RenewJobRequestAsync(It.IsAny<AgentJobRequestMessage>(), It.IsAny<ServiceEndpoint>(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token);
Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed.");
Assert.False(cancellationTokenSource.IsCancellationRequested); Assert.False(cancellationTokenSource.IsCancellationRequested);
@@ -389,9 +312,8 @@ namespace GitHub.Runner.Common.Tests.Listener
var jobDispatcher = new JobDispatcher(); var jobDispatcher = new JobDispatcher();
jobDispatcher.Initialize(hc); jobDispatcher.Initialize(hc);
// Act // Act
await jobDispatcher.RenewJobRequestAsync(It.IsAny<AgentJobRequestMessage>(), It.IsAny<ServiceEndpoint>(), 0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); await jobDispatcher.RenewJobRequestAsync(0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token);
// Assert // Assert
_configurationStore.Verify(x => x.SaveSettings(It.Is<RunnerSettings>(settings => settings.AgentName == newName)), Times.Once); _configurationStore.Verify(x => x.SaveSettings(It.Is<RunnerSettings>(settings => settings.AgentName == newName)), Times.Once);
@@ -446,7 +368,7 @@ namespace GitHub.Runner.Common.Tests.Listener
jobDispatcher.Initialize(hc); jobDispatcher.Initialize(hc);
// Act // Act
await jobDispatcher.RenewJobRequestAsync(It.IsAny<AgentJobRequestMessage>(), It.IsAny<ServiceEndpoint>(), 0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); await jobDispatcher.RenewJobRequestAsync(0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token);
// Assert // Assert
_configurationStore.Verify(x => x.SaveSettings(It.IsAny<RunnerSettings>()), Times.Never); _configurationStore.Verify(x => x.SaveSettings(It.IsAny<RunnerSettings>()), Times.Never);
@@ -499,7 +421,7 @@ namespace GitHub.Runner.Common.Tests.Listener
jobDispatcher.Initialize(hc); jobDispatcher.Initialize(hc);
// Act // Act
await jobDispatcher.RenewJobRequestAsync(It.IsAny<AgentJobRequestMessage>(), It.IsAny<ServiceEndpoint>(), 0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); await jobDispatcher.RenewJobRequestAsync(0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token);
// Assert // Assert
_configurationStore.Verify(x => x.SaveSettings(It.IsAny<RunnerSettings>()), Times.Never); _configurationStore.Verify(x => x.SaveSettings(It.IsAny<RunnerSettings>()), Times.Never);
@@ -557,7 +479,7 @@ namespace GitHub.Runner.Common.Tests.Listener
var jobDispatcher = new JobDispatcher(); var jobDispatcher = new JobDispatcher();
jobDispatcher.Initialize(hc); jobDispatcher.Initialize(hc);
await jobDispatcher.RenewJobRequestAsync(It.IsAny<AgentJobRequestMessage>(), It.IsAny<ServiceEndpoint>(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token);
Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed.");
Assert.True(cancellationTokenSource.IsCancellationRequested); Assert.True(cancellationTokenSource.IsCancellationRequested);
@@ -614,7 +536,7 @@ namespace GitHub.Runner.Common.Tests.Listener
var jobDispatcher = new JobDispatcher(); var jobDispatcher = new JobDispatcher();
jobDispatcher.Initialize(hc); jobDispatcher.Initialize(hc);
await jobDispatcher.RenewJobRequestAsync(It.IsAny<AgentJobRequestMessage>(), It.IsAny<ServiceEndpoint>(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token);
Assert.False(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should failed."); Assert.False(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should failed.");
Assert.False(cancellationTokenSource.IsCancellationRequested); Assert.False(cancellationTokenSource.IsCancellationRequested);
@@ -678,7 +600,7 @@ namespace GitHub.Runner.Common.Tests.Listener
var jobDispatcher = new JobDispatcher(); var jobDispatcher = new JobDispatcher();
jobDispatcher.Initialize(hc); jobDispatcher.Initialize(hc);
await jobDispatcher.RenewJobRequestAsync(It.IsAny<AgentJobRequestMessage>(), It.IsAny<ServiceEndpoint>(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token);
Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed.");
Assert.False(cancellationTokenSource.IsCancellationRequested); Assert.False(cancellationTokenSource.IsCancellationRequested);
@@ -737,78 +659,5 @@ namespace GitHub.Runner.Common.Tests.Listener
Assert.True(jobDispatcher.RunOnceJobCompleted.Task.Result, "JobDispatcher should set task complete token to 'TRUE' for one time agent."); Assert.True(jobDispatcher.RunOnceJobCompleted.Task.Result, "JobDispatcher should set task complete token to 'TRUE' for one time agent.");
} }
} }
private static void EnableRunServiceJobForJobDispatcher(JobDispatcher jobDispatcher)
{
// Set the value of the _isRunServiceJob field to true
var isRunServiceJobField = typeof(JobDispatcher).GetField("_isRunServiceJob", BindingFlags.NonPublic | BindingFlags.Instance);
isRunServiceJobField.SetValue(jobDispatcher, true);
}
private static ServiceEndpoint GetServiceEndpoint()
{
var serviceEndpoint = new ServiceEndpoint
{
Authorization = new EndpointAuthorization
{
Scheme = EndpointAuthorizationSchemes.OAuth
}
};
serviceEndpoint.Authorization.Parameters.Add("AccessToken", "token");
return serviceEndpoint;
}
private static AgentJobRequestMessage GetAgentJobRequestMessage()
{
var message = new AgentJobRequestMessage(
new TaskOrchestrationPlanReference()
{
PlanType = "Build",
PlanId = Guid.NewGuid(),
Version = 1
},
new TimelineReference()
{
Id = Guid.NewGuid()
},
Guid.NewGuid(),
"jobDisplayName",
"jobName",
null,
null,
new List<TemplateToken>(),
new Dictionary<string, VariableValue>()
{
{
"variables",
new VariableValue()
{
IsSecret = false,
Value = "variables"
}
}
},
new List<MaskHint>()
{
new MaskHint()
{
Type = MaskType.Variable,
Value = "maskHints"
}
},
new JobResources(),
new DictionaryContextData(),
new WorkspaceOptions(),
new List<JobStep>(),
new List<string>()
{
"fileTable"
},
null,
new List<TemplateToken>(),
new ActionsEnvironmentReference("env")
);
return message;
}
} }
} }

View File

@@ -502,34 +502,5 @@ namespace GitHub.Runner.Common.Tests.Listener
_messageListener.Verify(x => x.DeleteMessageAsync(It.IsAny<TaskAgentMessage>()), Times.Once()); _messageListener.Verify(x => x.DeleteMessageAsync(It.IsAny<TaskAgentMessage>()), Times.Once());
} }
} }
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Runner")]
public async void TestRemoveLocalRunnerConfig()
{
using (var hc = new TestHostContext(this))
{
hc.SetSingleton<IConfigurationManager>(_configurationManager.Object);
hc.SetSingleton<IConfigurationStore>(_configStore.Object);
hc.SetSingleton<IPromptManager>(_promptManager.Object);
var command = new CommandSettings(hc, new[] { "remove", "--local" });
_configStore.Setup(x => x.IsConfigured())
.Returns(true);
_configStore.Setup(x => x.HasCredentials())
.Returns(true);
var runner = new Runner.Listener.Runner();
runner.Initialize(hc);
await runner.ExecuteCommand(command);
// verify that we delete the local runner config with the correct remove parameter
_configurationManager.Verify(x => x.DeleteLocalRunnerConfig(), Times.Once());
}
}
} }
} }

Some files were not shown because too many files have changed in this diff Show More