Compare commits

...

127 Commits

Author SHA1 Message Date
Thomas Boop
5effa808be Release v2.276.1 2021-01-21 14:27:46 -05:00
Thomas Boop
88098a6705 Update Runner 276 release branch (#931)
* Update ldd check with dotnet 5.

* Runner v2.276.0 fixes (#928)

* Revert "always use Fips Cryptography (#896)"

3b34e203dc

* Revert "Update ldd check with dotnet 5."

4b6ded0a01

* Revert "Update SDK to .NET 5 (#799)"

fc3ca9bb92

* Update dotnet-install scripts

* prep 2.276.1 runner release (#929)

Co-authored-by: Tingluo Huang <huangtingluo@gmail.com>
2021-01-21 14:26:33 -05:00
Tingluo Huang
2ee7717774 Update ldd check with dotnet 5. 2021-01-15 09:18:37 -05:00
Tingluo Huang
c946435010 release 2.276.0 runner. 2021-01-14 13:56:22 -05:00
TingluoHuang
0953ffa62b Prepare 2.276.0 runner release. 2021-01-14 13:55:13 -05:00
Robin Neatherway
66727f76c8 Add on: pull_request trigger to CodeQL workflow (#907)
From February 2021, in order to provide feedback on pull requests, Code Scanning workflows must be configured with both `push` and `pull_request` triggers. This is because Code Scanning compares the results from a pull request against the results for the base branch to tell you only what has changed between the two.

Early in the beta period we supported displaying results on pull requests for workflows with only `push` triggers, but have discontinued support as this proved to be less robust.

See https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#scanning-pull-requests for more information on how best to configure your Code Scanning workflows.
2021-01-14 13:26:37 -05:00
Tingluo Huang
7ee333b5cd Adding --check to run a serials network test against GitHub or GHES. (#900)
* add --check.
2021-01-14 13:26:07 -05:00
Tingluo Huang
3b34e203dc always use Fips Cryptography (#896) 2021-01-12 11:05:01 -05:00
Tingluo Huang
e808190dd2 add warning when running out of disk. (#873) 2021-01-05 21:49:11 -05:00
Yang Cao
d2cb9d7685 Ignore certain scenarios so they are not counted as infra failures (#889)
* Ignore certain scenarios so they are not counted as infra failures

* Check to make sure request is not cancelled
2021-01-05 16:08:02 -05:00
Tiago Silva
5ba6a2c78d Add new ANDROID_SDK_ROOT environment variable (#892) 2021-01-05 13:39:47 -05:00
Adeel Mujahid
fc3ca9bb92 Update SDK to .NET 5 (#799) 2020-12-26 00:01:45 -05:00
Tingluo Huang
a94a19bb36 add e2e workflow badge 2020-12-25 22:39:25 -05:00
Tingluo Huang
a9be5f6557 Update job name in logs. 2020-12-25 22:17:24 -05:00
Tingluo Huang
3600f20cd3 e2e test workflow (#885) 2020-12-25 21:54:46 -05:00
Tingluo Huang
81a00fff3e config runner via PAT. (#874) 2020-12-25 21:54:23 -05:00
Josh Gross
31474098ff Add runtime team as CODEOWNERS (#872) 2020-12-18 14:21:15 -05:00
David Kale
7ff6ff6afa Prepare 2.275.1 2020-12-14 16:36:31 -05:00
Tingluo Huang
56529a1c2f fix compat issue in timeline record state. (#861) 2020-12-14 15:43:00 -05:00
David Kale
510fadf71a Prepare m275 (#860) 2020-12-14 11:02:44 -05:00
klassiker
007ac8138b Add proxy support for container actions (#840)
* Add proxy support for container actions in Runner.Worker/StepsRunner

* Move proxy modifications to ContainerActionHandler
2020-12-11 13:08:45 -05:00
Yang Cao
1e12b8909a Count actions resolve failures as infra failures (#851)
During job run we may fail to resolve actions download info, and this
stack is fully controlled by GitHub actions so it should be counted as
infrastructure failure instead of user failure.
2020-12-11 11:07:43 -05:00
Tingluo Huang
9ceb3d481a unset GTIHUB_ACTION_REPOSITORY and GITHUB_ACTION_REF for non-repo based actions. (#804) 2020-12-11 11:04:07 -05:00
Bruno FERNANDO
3bce2eb09c feat(scripts): add labels in the script that register runner (#844) 2020-12-11 11:03:04 -05:00
David Kale
80bf68db81 Crypto cleanup and enable usage of FIPS compliant crypto when required (#806)
* Use FIPS compliant crypto when required

* Comment cleanup

* Store OAuth signing scheme in credentialData instead of runner setting

Add encryption scheme for job message encyption key to session

Further cleanup of unused crypto code

* Update windows rsa key manager to use crossplat dotnet RSA api

* Undo unneeded ConfigurationManager change
2020-12-04 11:35:16 -05:00
Thomas Boop
a2e32170fd Disable set-env and add-pathcommands (#779)
* Disable Old Runner Commands set-env and add-path

* update dotnet install scripts

* update runner version and release notes
2020-11-16 08:20:43 -05:00
Thomas Boop
35dda19491 Add deprecation date and release 2.274.1 version (#796) 2020-11-09 09:01:47 -05:00
Julio Barba
36bdf50bc6 Prepare the release of 2.274.0 runner 2020-11-05 10:25:24 -05:00
Chris Gavin
95e2158dc6 Add an environment variable to indicate which repository the currently running Action came from. (#585)
* add `workflow_dispatch`

* Add an environment variable to indicate which repository the currently running Action came from.

* Expose the Action ref as well.

* Move setting `github.action_repository` and `github.action_ref` to `ActionRunner.cs`.

* Don't set `action_repository` and `action_ref` for local Actions.

Co-authored-by: Tingluo Huang <tingluohuang@github.com>
2020-11-03 14:39:17 -05:00
Jason Laqua
3ebaeb9f19 Fixes #759 doesn't change proxy environment variables (#760)
* Fixes #759 doesn't change proxy environment variables

* Update RunnerWebProxy.cs

* Update RunnerWebProxyL0.cs

Co-authored-by: Tingluo Huang <tingluohuang@github.com>
2020-11-03 10:47:30 -05:00
shinriyo
9d678cb270 DRY and add sudo (#687)
remove 3 "redundant" text and put one text for DRY.
and developers always forget `sudo` and annoying `Need to run with sudo privilege` message.
so, add first.
2020-11-02 21:38:35 -05:00
Tingluo Huang
27788491ea raise error for set-env, block set node_options. (#784)
* raise error for set-env, block set node_options.

* feedback.
2020-11-02 14:09:29 -05:00
Yashwanth Anantharaju
5ba7affea4 fix in correct check (#778) 2020-10-30 14:34:00 -04:00
Robin Neatherway
ce92d7a6b5 Change ping .. > nul to sleep (#647)
* Change `ping .. > nul` to `sleep`

The filename `nul` is a Windows-ism that causes the update script to
create such a file in the current working directory. The `ping`
utility is also an dependency not installed by
`installdependencies.sh`, so it seemed easier to change it to the
standard `sleep` command.

* Update dotnet-install script as requested by test

* Update dotnet-install.ps1

Co-authored-by: Tingluo Huang <tingluohuang@github.com>
2020-10-29 10:15:30 -04:00
Temtaime
d23ca0ba7a Add .editorconfig (#768)
* Add .editorconfig

* Create .editorconfig
2020-10-27 10:50:50 -04:00
dependabot[bot]
9d1c81f018 Bump @actions/core in /src/Misc/expressionFunc/hashFiles (#729)
Bumps [@actions/core](https://github.com/actions/toolkit/tree/HEAD/packages/core) from 1.2.0 to 1.2.6.
- [Release notes](https://github.com/actions/toolkit/releases)
- [Changelog](https://github.com/actions/toolkit/blob/main/packages/core/RELEASES.md)
- [Commits](https://github.com/actions/toolkit/commits/HEAD/packages/core)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2020-10-26 23:29:08 -04:00
Josh Soref
7a8abe726a Improve apt handling (#708)
* Unify apt/apt-get logic

The previous logic was buggy in that it tried to use `apt` in the `apt-get` branch after deciding that `apt` was unavailable...

* Prefer apt-get over apt

apt does not have a stable cli and using it from scripts yields annoying messages

* Improve English for missing apt-get & apt case

* Fix apt-get/apt fallback behavior for $ patterns

If there's a `$` in the apt install pattern, it will not fail if it selects a thing and decides it isn't interested in installing it.

* Fix spelling of libssl
2020-10-26 23:27:09 -04:00
Łukasz Łaniewski-Wołłk
a9135e61a0 Correcting bug in check of libicu presence (#695) 2020-10-26 23:14:17 -04:00
Justin Weissig
feafd3e1d7 fixed grammar issues (#672)
Nothing major here just minor wording.
2020-10-26 23:11:30 -04:00
Justin Weissig
dc3b2d3a36 fixed wording (#671)
Fixed a few minor grammar issues
2020-10-26 23:10:59 -04:00
Justin Weissig
a371309079 minor spelling & grammar tweaks (#670)
Fixed a few minor spelling & grammar issues.
2020-10-26 23:10:26 -04:00
Justin Weissig
5dd6bde4ca fixed minor spelling mistake (#669)
Changed enhancment to enhancement.
2020-10-26 23:09:38 -04:00
Tingluo Huang
c196103e58 update dotnet install script. 2020-10-26 23:07:57 -04:00
Fabian Mastenbroek
d55070da3e Update to .NET Core SDK 3.1.302 (#681)
This change updates the .NET Core SDK used by the Actions Runner to
version 3.1.302 to address the issues that are caused by the following issue:
    https://github.com/dotnet/runtime/issues/13475
See #574 for more information.

Fixes #574
2020-10-26 22:51:29 -04:00
Yashwanth Anantharaju
8279ae9a70 Support environment URL parsing (#762)
* environment URL parsing
2020-10-21 12:14:21 -04:00
Hayden Faulds
2e3b03623f log runner group name (#696)
* log runner group name

* linting
2020-10-16 14:56:06 +01:00
Thomas Boop
c18c8746db Release notes for 2.273.5 (#734) 2020-10-02 11:49:49 -04:00
Thomas Boop
6332a52d76 Notify on unsecure commands (#731)
* notify on unsecure commands
2020-10-02 11:34:37 -04:00
Yang Cao
8bb588bb69 Expose retention days in env for toolkit/artifacts package (#714) 2020-09-17 15:11:12 -04:00
David Kale
4510f69c73 Prepare 273.4 release 2020-09-17 18:19:42 +00:00
David Kale
c7b8552edf Prepare 2.273.3 release 2020-09-16 15:06:07 +00:00
Julio Barba
0face6e3af Preparing the release of 2.273.2 runner 2020-09-14 13:06:41 -04:00
eric sciple
306be41266 fix bug w checkout v1 updating GITHUB_WORKSPACE (#704) 2020-09-14 12:00:00 -04:00
David Kale
4e85b8f3b7 Allow registry credentials for job/service containers (#694)
* Log in with container credentials if given

* Stub in registry aware auth for later

* Fix hang if password is empty

* Remove default param to fix build

* PR Feedback. Add some tests and fix parse
2020-09-11 12:28:58 -04:00
Julio Barba
444332ca88 Prepare the release of 2.273.1 runner 2020-09-08 13:01:36 -04:00
Thomas Boop
e6eb9e381d Cleanup FileCommands (#693) 2020-09-04 15:35:36 -04:00
eric sciple
3a76a2e291 read env file (#683) 2020-08-29 23:18:35 -04:00
Thomas Boop
9976cb92a0 Add Runner File Commands (#684)
* Add File Runner Commands
2020-08-28 15:32:25 -04:00
Thomas Brumley
d900654c42 Add in Log line numbers for streaming logs (#663)
* Add in Log line

Co-authored-by: yaananth (Yash) <yaananth@github.com>
2020-08-25 12:02:29 -04:00
Julio Barba
65e3ec86b4 Set executable bit 2020-08-18 16:09:04 -04:00
Julio Barba
a7f205593a Update dotnet scripts 2020-08-18 16:03:06 -04:00
Julio Barba
55f60a4ffc Prepare the release of 2.273.0 runner 2020-08-17 15:41:15 -04:00
Ethan Chiu
ca13b25240 Fix Outputs Example (#658) 2020-08-14 11:55:27 -04:00
Timo Schilling
b0c2734380 fix endgroup maker (#640) 2020-08-14 11:53:30 -04:00
Ethan Chiu
9e7b56f698 Fix Null Ref Issues Composite Actions (#657) 2020-08-12 17:12:54 -04:00
Ethan Chiu
8c29e33e88 Fix DisplayName Changing in middle of composite action run (#645) 2020-08-10 16:26:23 -04:00
Ethan Chiu
976217d6ec Free up memory from step level outputs in composite action (#641) 2020-08-10 14:31:30 -04:00
Ethan Chiu
562eafab3a Adding Documentation to ADR for Support for Script Execution + Explicit Definition (#616) 2020-08-10 11:30:34 -04:00
Joe Bourne
9015b95a72 Updating virtual environment terminology (#651)
* Dropping pool terminology

* Update README.md
2020-08-07 15:52:56 -04:00
eric sciple
7d4bbf46de fix feature flag check; omit context for generated context names (#638) 2020-08-04 11:12:40 -04:00
Christopher Johnson
7b608e3e92 Adding help text for the new runnergroup feature (#626)
Co-authored-by: Christopher Johnson <thchrisjohnson@github.com>
2020-07-30 12:03:40 -04:00
Ethan Chiu
f028b4e2b0 Revert JobSteps to Queue Data Structure (#625)
* Revert JobSteps to Queue data structure

* Revert tests
2020-07-29 16:19:04 -04:00
TingluoHuang
38f816c2ae prepare release 2.272.0 runner. 2020-07-29 15:31:45 -04:00
efyx
bc1fe2cfe0 Fix poor performance of process spawned from svc daemon (#614) 2020-07-29 15:20:28 -04:00
Ethan Chiu
89a13db2c3 Remove TESTING_COMPOSITE_ACTIONS_ALPHA Env Variable (#624) 2020-07-29 15:12:15 -04:00
Ethan Chiu
d59092d973 GITHUB_ACTION_PATH + GITHUB_ACTION so that we can run scripts for Composite Run Steps (#615)
* Add environment variable for GITHUB_ACTION_PATH

* ah

* Remove debugging messages

* Set github action path at step level instead of global scope to avoid necessary removal

* Remove set context for github action

* Set github action path before and after composite action

* Copy GitHub Context, use this copied context for each composit step, and then set the action_path for each one (to avoid stamping over parent pointer GitHubContext
2020-07-29 14:28:14 -04:00
Ethan Chiu
855b90c3d4 Explicitly define what is allowed for a composite action (#605)
* Explicitly define what is allowed for an action

* Add step-env

* Remove secrets + defaults

* new line

* Add safety check to prevent from checking defaults in ScriptHandler for composite action

* Revert "Add safety check to prevent from checking defaults in ScriptHandler for composite action"

This reverts commit aeae15de7b.

* Need to explictly use ActionStep type since we need the .Inputs attribute which is only found in the ActionStep not IStep

* Fix ActionManifestManager

* Remove todos

* Revert "Revert "Add safety check to prevent from checking defaults in ScriptHandler for composite action""

This reverts commit a22fcbc036.

* revert

* Remove needs in env

* Make shell required + add inputs

* Remove passing context to all composite steps attribuyte
2020-07-28 10:15:46 -04:00
Christopher Johnson
48ac96307c Add ability to register a runner to the non-default self-hosted runner group (#613)
Co-authored-by: Christopher Johnson <thchrisjohnson@github.com>
2020-07-23 17:46:48 -04:00
Ethan Chiu
2e50dffb37 Clean Up Composite UI (#610)
* Remove redundant code (display name is already evaluated in ActionRunner beforehand for each step)

* remove

* Remove nesting information for composite steps.

* put messages in debug logs if composite. if not, put these messages as outputs

* Fix group issue

* Fix end group issue
2020-07-23 09:45:00 -04:00
Tingluo Huang
e7b0844772 Add manual trigger 2020-07-23 00:34:36 -04:00
Ethan Chiu
d5a5550649 Fix Timeout-minutes for Whole Composite Action Step (#599)
* Exploring child Linked Cancellation Tokens

* Preliminary Timeout-minutes fix

* Final Solution for resolving cancellation token's timeout vs. cancellation

* Clean up + Fix error handling

* Use linked tokens instead

* Clean up

* one liner

* Remove JobExecutionContext => Replace with public Root accessor

* Move CreateLinkedTokenSource in the CreateCompositeStep Function
2020-07-22 18:01:50 -04:00
Ethan Chiu
3d0147d322 Improve Debugging Messages for Empty Tokens (#609)
* Improve Debugging Messages for Empty Tokens

* fix tests
2020-07-22 17:34:40 -04:00
Ethan Chiu
bd1f245aac Clarify details for defaults, shell, and working-dir (#607) 2020-07-22 16:11:55 -04:00
Steven Maude
005f1c15b1 Fix "propogate" typo in ADR 0549 (#600) 2020-07-22 16:10:57 -04:00
David Kale
da3cb5506f Fold logs for intermediate docker commands (#608) 2020-07-22 14:55:49 -04:00
dependabot[bot]
32d439070b Bump lodash in /src/Misc/expressionFunc/hashFiles (#603)
Bumps [lodash](https://github.com/lodash/lodash) from 4.17.15 to 4.17.19.
- [Release notes](https://github.com/lodash/lodash/releases)
- [Commits](https://github.com/lodash/lodash/compare/4.17.15...4.17.19)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2020-07-20 10:21:01 -04:00
jeffrey
ec9f8f1682 dbl quotes around variable so CD works if path contains spaces (#602) 2020-07-20 10:19:37 -04:00
eric sciple
0921af735a move shared ExecutionContext properties under .Global (#594) 2020-07-19 19:05:47 -04:00
eric sciple
1cc3c08cf2 Prepare to switch GITHUB_ACTION to use ContextName instead of refname (#593)
This PR changes GITHUB_ACTION to use the step ContextName, instead of refname. The behavior is behind a feature flag. Refname is an otherwise deprecated property.

Primary motivation: For composite actions, we need a distinct GITHUB_ACTION for each nested step. This PR adds code to generate a default context name for nested steps.

For nested steps, GITHUB_ACTION will be set to "{ScopeName}.{ContextName}" to ensure no collisions.

A corresponding change will be made on the server so context name is never empty. Generated context names will start with "__".

A follow-up PR is required to avoid tracking "step" context values (outputs/conclusion/result) for generated context names. Waiting on telemetry from the server to confirm it's safe to assume leading "__" is a generate context name.
2020-07-19 17:19:13 -04:00
Ethan Chiu
f9dca15c63 Composite Run Steps Refactoring (#591)
* Add basic framework for baby steps runner

* Basic logic for adding steps / invoking composite action steps

* Composite Steps Runner MVP

* Fix null object reference error

* intialize composiute

* Comment out code that is handled by stepsrunner

* Add composite clean up step

* Remove previous 'workarounds' from StepsRunner. Clean Up PR

* Remove todo

* Remove todo

* Fix using unitialized object yikes

* Remove time delay

* Format handler

* Move output handler into action handler

* Add try to evaluate display name

* Remove while loop yikes

* Abstract away the windows encoding check during step running

* Github context set to {ScopeName}.{ContextName} or {ContextName} if ScopeName is null

* Remove setting result to sucess since result defaults to sucess

* Fix windows error

* Fix windows

* revert:

* Windows fix

* Fix Windows Error in Abstraction

* Remove Composite Steps Runner => consolidate into Composite Steps Runner

* Remove unn. attribute in ExecutionContext

* Change protection levels, plus change function name to more clear meaning

* Remove location param

* location pt.2 fix

* Remove outputs step

* Remove temp directory

* new line

* Add arguitl not null

* better comment

* Change encoding name

* Check count > 0 for composite steps, import System.Threading

* Change function header encodingutil

* Add TODO

* Add await

* Handle Failed Step

* Move over SetAllCompositeOutputs to the handler

* Remove timeout-minutes setting in steps-level

* Use only ExecutionContext

* Move using to the top

* Remove redundant check

* Change function name

* Remove testing code

* Consolidate error code

* Consolidate code

* Change HandleOutput => ProcessCompositeActionOutputs

* Remove set the timeout comment

* Add Cancelling functionality + Remove unn. parameter
2020-07-17 16:31:48 -04:00
eric sciple
0877d9a533 Update StringUtil.cs 2020-07-16 10:30:42 -04:00
eric sciple
d5e40c6a60 Update 0549-composite-run-steps.md 2020-07-15 20:00:45 -04:00
eric sciple
391bc35bb9 Update 0549-composite-run-steps.md 2020-07-15 19:59:13 -04:00
eric sciple
e4267b8434 Update 0549-composite-run-steps.md 2020-07-15 19:57:22 -04:00
TingluoHuang
2709cbc0ea rename master to main. 2020-07-14 13:37:20 -04:00
Ethan Chiu
5e0cde8649 Composite Actions UI (#578)
* Composite Action Run Steps

* Env Flow => Able to get env variables and overwrite current env variables => but it doesn't 'stick'

* clean up

* Clean up trace messages + add Trace debug in ActionManager

* Add debugging message

* Optimize runtime of code

* Change String to string

* Add comma to Composite

* Change JobSteps to a List, Change Register Step function name

* Add TODO, remove unn. content

* Remove unnecessary code

* Fix unit tests

* Fix env format

* Remove comment

* Remove TODO message for context

* Add verbose trace logs which are only viewable by devs

* Initial Start for FileTable stuff

* Progress towards passing FileTable or FileID or FileName

* Sort usings in Composite Action Handler

* Change 0 to location

* Update context variables in composite action yaml

* Add helpful error message for null steps

* Pass fileID to all children token of root action token

* Change confusing term context => templateContext, Eliminate _fileTable and only use ExecutionContext.FileTable + update this table when need be

* Remove unnessary FileID attribute from CompositeActionExecutionData

* Clean up file path for error message

* Remove todo

* Initial start/framework for output handling

* Outline different class vs Handler approach

* Remove InitializeScope

* Remove InitializeScope

* Fix Workflow Step Env overiding Parent Env

* First Approach for Attaching ID + Group ID to each Composite Action Step

* Add GroupID to the ActionDefinitionData

* starting foundation for handling clean up outputs step

* Pass outputs data to each composite action step to enable set-output functionality

* Create ScopeName for whole composite action.

This will enable us to add to the StepsContext[ScopeName] for the composite action which will allow us to use all these outputs in the cleanup step

* Hook up composite output step to handler => tmmrw implement composite output handler

* Add post composite action step to cleanup outputs => triggers composite output cleanup handler

* Fix Outputs Token handling start. Add individual step scope names.

* Set up Scope Name and Context Name naming system{

* Figured out how to pass Parent Execution Context to clean up step

* Figured out how to pass Parent Execution Context and scope names to
clean up step

* Add GetOutput function for StepsContext

* Generate child scope name correctly if parent scope name is null

* Simplify InitializeScope()

* Outputs are set correctly and able to get all final outputs in handler

* Parse through Action Outputs

* Fix null ScopeName + ContextName in CompositeOutputHandler

* Shift over handling of Action Outputs to output handler

* First attempt to fix null retrievals for output variables

* Basic Support for Outputs Done.

* Clean up pt.1

* Refactor outputs to avoid using Action Reference

* Clean up code

* Clean up part 2

* Add clarifying comments for the output handler

* Remove TODO

* Remove env in composite action scope

* Clean up

* Revert back

* revert back

* add back envToken

* Remove unnecessary code

* Add file length check

* Clean up

* Fix logging issue

* Figure out how to handle set-env edge cases

* formatting

* fix unit tests

* Fix windows unit test syntax error

* Fix period

* Sanity check for fileTable add + remove unn. code

* revert back

* Add back line break

* Fix null errors

* Address situation if FileTable is null + add sanity check for adding file to fileTable

* add line

* Revert

* Fix unit tests to instantiate a FileTable

* Fix logic for trimming manifestfile path

* Add null check

* Revert

* Revert

* revert

* spacing

* Add filetable to testing file, remove ? since we know filetable should never be non null

* Fix Throw logic

* Clarify template outputs token

* Add another type support for outputs to avoid container unit tests errors

* Add mapping for parity

* Build support for new outputs format

* Build support for new outputs format

* Refactor to avoid duplication of action yaml for workflow yaml

* revert

* revert

* revert

* spacing
2020-07-13 17:55:15 -04:00
Ethan Chiu
cb2b323781 Composite Run Steps Outputs (#568)
* Composite Action Run Steps

* Env Flow => Able to get env variables and overwrite current env variables => but it doesn't 'stick'

* clean up

* Clean up trace messages + add Trace debug in ActionManager

* Add debugging message

* Optimize runtime of code

* Change String to string

* Add comma to Composite

* Change JobSteps to a List, Change Register Step function name

* Add TODO, remove unn. content

* Remove unnecessary code

* Fix unit tests

* Fix env format

* Remove comment

* Remove TODO message for context

* Add verbose trace logs which are only viewable by devs

* Initial Start for FileTable stuff

* Progress towards passing FileTable or FileID or FileName

* Sort usings in Composite Action Handler

* Change 0 to location

* Update context variables in composite action yaml

* Add helpful error message for null steps

* Pass fileID to all children token of root action token

* Change confusing term context => templateContext, Eliminate _fileTable and only use ExecutionContext.FileTable + update this table when need be

* Remove unnessary FileID attribute from CompositeActionExecutionData

* Clean up file path for error message

* Remove todo

* Initial start/framework for output handling

* Outline different class vs Handler approach

* Remove InitializeScope

* Remove InitializeScope

* Fix Workflow Step Env overiding Parent Env

* First Approach for Attaching ID + Group ID to each Composite Action Step

* Add GroupID to the ActionDefinitionData

* starting foundation for handling clean up outputs step

* Pass outputs data to each composite action step to enable set-output functionality

* Create ScopeName for whole composite action.

This will enable us to add to the StepsContext[ScopeName] for the composite action which will allow us to use all these outputs in the cleanup step

* Hook up composite output step to handler => tmmrw implement composite output handler

* Add post composite action step to cleanup outputs => triggers composite output cleanup handler

* Fix Outputs Token handling start. Add individual step scope names.

* Set up Scope Name and Context Name naming system{

* Figured out how to pass Parent Execution Context to clean up step

* Figured out how to pass Parent Execution Context and scope names to
clean up step

* Add GetOutput function for StepsContext

* Generate child scope name correctly if parent scope name is null

* Simplify InitializeScope()

* Outputs are set correctly and able to get all final outputs in handler

* Parse through Action Outputs

* Fix null ScopeName + ContextName in CompositeOutputHandler

* Shift over handling of Action Outputs to output handler

* First attempt to fix null retrievals for output variables

* Basic Support for Outputs Done.

* Clean up pt.1

* Refactor outputs to avoid using Action Reference

* Clean up code

* Clean up part 2

* Add clarifying comments for the output handler

* Remove TODO

* Remove env in composite action scope

* Clean up

* Revert back

* revert back

* add back envToken

* Remove unnecessary code

* Add file length check

* Clean up

* Figure out how to handle set-env edge cases

* formatting

* fix unit tests

* Fix windows unit test syntax error

* Fix period

* Sanity check for fileTable add + remove unn. code

* revert back

* Add back line break

* Fix null errors

* Address situation if FileTable is null + add sanity check for adding file to fileTable

* add line

* Revert

* Fix unit tests to instantiate a FileTable

* Fix logic for trimming manifestfile path

* Add null check

* Revert

* Revert

* revert

* spacing

* Add filetable to testing file, remove ? since we know filetable should never be non null

* Fix Throw logic

* Clarify template outputs token

* Add another type support for outputs to avoid container unit tests errors

* Add mapping for parity

* Build support for new outputs format

* Refactor to avoid duplication of action yaml for workflow yaml

* Move SDK work in ActionManifestManager, Condense Code

* Defer runs evaluation till after for loop to ensure order doesn't matter

* Fix logic error in setting scope and context names

* Add Regex + Add Child Context name null resolution

* move private function to bottom of class
2020-07-13 17:23:19 -04:00
Ethan Chiu
6c3958f365 Composite Run Steps ADR (#554)
* start

* Inputs + Outputs

* Clarify docs

* Finish Environment

* Add if condition

* Clarify language

* Update 0549-composite-run-steps.md

* timeout-minutes

* Finish

* add relevant example

* Fix syntax

* fix env example

* fix yaml syntax

* Update 0549-composite-run-steps.md

* Update file names, add more relevant example if condition

* Add note to continue-on-error

* Apply changes to If Condition

* bolding

* Update 0549-composite-run-steps.md

* Update 0549-composite-run-steps.md

* Update 0549-composite-run-steps.md

* Update 0549-composite-run-steps.md

* Syntax support + spacing

* Add guiding principles.

* Update 0549-composite-run-steps.md

* Reverse order.

* Update 0549-composite-run-steps.md

* change from job to step

* Update 0549-composite-run-steps.md

* Update 0549-composite-run-steps.md

* Update 0549-composite-run-steps.md

* Add Secrets

* Update 0549-composite-run-steps.md

* Update 0549-composite-run-steps.md

* Fix output example

* Fix output example

* Fix action examples to use using.

* fix output variable name

* update workingDir + env

* Defaults + continue-on-error

* Update Outputs Section

* Eliminate Env

* Secrets

* Update timeout-minutes

* Update 0549-composite-run-steps.md

* Update 0549-composite-run-steps.md

* Fix example.

* Remove TODOs

* Update 0549-composite-run-steps.md

* Update 0549-composite-run-steps.md
2020-07-13 12:30:31 -04:00
Ethan Chiu
9d7bd4706b Improve Error Messaging for Actions by Using ExecutionContext's FileTable as Single Source of Truth and by Passing FileID to All Children Tokens. (#564)
* Composite Action Run Steps

* Env Flow => Able to get env variables and overwrite current env variables => but it doesn't 'stick'

* clean up

* Clean up trace messages + add Trace debug in ActionManager

* Add debugging message

* Optimize runtime of code

* Change String to string

* Add comma to Composite

* Change JobSteps to a List, Change Register Step function name

* Add TODO, remove unn. content

* Remove unnecessary code

* Fix unit tests

* Fix env format

* Remove comment

* Remove TODO message for context

* Add verbose trace logs which are only viewable by devs

* Initial Start for FileTable stuff

* Progress towards passing FileTable or FileID or FileName

* Sort usings in Composite Action Handler

* Change 0 to location

* Update context variables in composite action yaml

* Add helpful error message for null steps

* Pass fileID to all children token of root action token

* Change confusing term context => templateContext, Eliminate _fileTable and only use ExecutionContext.FileTable + update this table when need be

* Remove unnessary FileID attribute from CompositeActionExecutionData

* Clean up file path for error message

* Remove todo

* Fix Workflow Step Env overiding Parent Env

* Remove env in composite action scope

* Clean up

* Revert back

* revert back

* add back envToken

* Remove unnecessary code

* Add file length check

* Clean up

* Figure out how to handle set-env edge cases

* formatting

* fix unit tests

* Fix windows unit test syntax error

* Fix period

* Sanity check for fileTable add + remove unn. code

* revert back

* Add back line break

* Fix null errors

* Address situation if FileTable is null + add sanity check for adding file to fileTable

* add line

* Revert

* Fix unit tests to instantiate a FileTable

* Fix logic for trimming manifestfile path

* Add null check

* Add filetable to testing file, remove ? since we know filetable should never be non null
2020-07-08 17:15:16 -04:00
Ethan Chiu
5822a38c39 Add bash command for running custom runner (#569) 2020-07-08 11:20:38 -04:00
Ethan Chiu
d42c9da2d7 Composite Actions: Support Env Flow (#557)
* Composite Action Run Steps

* Env Flow => Able to get env variables and overwrite current env variables => but it doesn't 'stick'

* clean up

* Clean up trace messages + add Trace debug in ActionManager

* Add debugging message

* Optimize runtime of code

* Change String to string

* Add comma to Composite

* Change JobSteps to a List, Change Register Step function name

* Add TODO, remove unn. content

* Remove unnecessary code

* Fix unit tests

* Fix env format

* Remove comment

* Remove TODO message for context

* Add verbose trace logs which are only viewable by devs

* Sort usings in Composite Action Handler

* Change 0 to location

* Update context variables in composite action yaml

* Add helpful error message for null steps

* Fix Workflow Step Env overiding Parent Env

* Remove env in composite action scope

* Clean up

* Revert back

* revert back

* add back envToken

* Remove unnecessary code

* Figure out how to handle set-env edge cases

* formatting

* fix unit tests

* Fix windows unit test syntax error
2020-07-08 10:16:51 -04:00
eric sciple
121deedeb5 Fix trailing '.0' for Int64 values (#572) 2020-06-30 17:25:47 -04:00
Ethan Chiu
a0942ed345 Composite Actions Support for Multiple Run Steps (#549)
* Composite Action Run Steps

* Clean up trace messages + add Trace debug in ActionManager

* Change String to string

* Add comma to Composite

* Change JobSteps to a List, Change Register Step function name

* Add TODO, remove unn. content

* Remove unnecessary code

* Fix unit tests

* Add verbose trace logs which are only viewable by devs

* Sort usings in Composite Action Handler

* Change 0 to location

* Update context variables in composite action yaml

* Add helpful error message for null steps
2020-06-23 15:35:32 -04:00
TingluoHuang
7cef9a27ca release 2.267.0 runner. 2020-06-23 14:05:28 -04:00
Tingluo Huang
df7e16954e print runner and machine name to log. (#539) 2020-06-23 13:57:37 -04:00
eric sciple
4e7d27a53c remove temporary logic when resolving action download info (#550) 2020-06-15 13:13:47 -04:00
Lokesh Gopu
89d1418e48 Update exception message (#540) 2020-06-11 17:25:50 -04:00
Tingluo Huang
e728b8594d fix race condition. (#538) 2020-06-11 16:17:24 -04:00
Tingluo Huang
de4490d06d Restore SELinux context on service file when SELinux is enabled (#525) 2020-06-11 15:40:09 -04:00
Tingluo Huang
2e800f857e Skip search $PATH on command with fully qualified path (#526) 2020-06-11 13:52:42 -04:00
Tingluo Huang
312c7668a8 Fix DataContract with Token service (#532) 2020-06-11 12:11:35 -04:00
Tingluo Huang
eaf39bb058 add libicu66 for Ubuntu 20.04 (#535) 2020-06-11 12:11:13 -04:00
eric sciple
5815819f24 Resolve action download info (#515) 2020-06-09 08:53:28 -04:00
Ethan Chiu
1aea046932 Add substep for developer flow for clarity (#523) 2020-06-08 10:47:58 -04:00
Ethan Chiu
eda463601c Update Links and Language to Git + VSCode (#522) 2020-06-08 10:19:17 -04:00
Nick Fields
f994ae0542 Reduce input validation warnings (#506)
* Only raise a single warning for unexpected inputs

* Update invalid input test to raise single warning
2020-06-05 23:09:14 -04:00
Tingluo Huang
3c5aef791c Fix null ref exception in SecretMasker caused by hashfiles timeout. (#516) 2020-06-05 23:02:10 -04:00
Tingluo Huang
c4626d0c3a Remove SPS/Token migration code. Remove GHES url manipulate code. (#513)
* Remove SPS/Token migration code. Remove GHES url manipulate code.

* feedback.
2020-06-03 23:24:53 -04:00
eric sciple
416a7ac4b8 prepare to switch to service resolves archive download info (#508) 2020-06-02 17:21:50 -04:00
TingluoHuang
11435857e4 prepare 2.263.0 runner release. 2020-05-21 15:49:10 -04:00
Tingluo Huang
6f260012a3 Fix inputs validation warning, fix post step display name, fix worker crash due to error in step.env (#490) 2020-05-21 11:09:50 -04:00
eric sciple
4fc87ddfc6 fix problem matcher for GHES (#488) 2020-05-19 16:15:03 -04:00
eric sciple
b45c1b9440 switch GITHUB_URL to GITHUB_SERVER_URL (#482) 2020-05-18 13:02:30 -04:00
Quan TRAN
73307c0a30 jq returns "null" if the field does not exist (#478) 2020-05-14 11:09:20 -04:00
Tingluo Huang
cd8e4ddba1 Validate inputs only for repo action, no warning for small delay. (#476)
* validate inputs only for repo action, no warning for small delay.

* l0
2020-05-12 16:09:13 -04:00
Tingluo Huang
abf59bdcb6 Fix configure as service with runner name has space. (#474) 2020-05-12 16:08:50 -04:00
Brian Cristante
09cf59c1e0 Use an env var to point to an Actions Service dev instance (#468)
* Use an environment variable for the testing backdoor

* Make the env var a boolean

We'll still have to pass the URL on the command line

* Empty commit to rerun CI
2020-05-11 17:14:02 -04:00
144 changed files with 8574 additions and 3373 deletions

View File

@@ -1,9 +1,10 @@
name: Runner CI name: Runner CI
on: on:
workflow_dispatch:
push: push:
branches: branches:
- master - main
- releases/* - releases/*
paths-ignore: paths-ignore:
- '**.md' - '**.md'

View File

@@ -2,6 +2,7 @@ name: "Code Scanning - Action"
on: on:
push: push:
pull_request:
schedule: schedule:
- cron: '0 0 * * 0' - cron: '0 0 * * 0'

335
.github/workflows/e2etest.yml vendored Normal file
View File

@@ -0,0 +1,335 @@
name: Runner E2E Test
on:
workflow_dispatch:
push:
branches:
- main
- releases/*
jobs:
init:
name: Initialize workflow ☕
runs-on: ubuntu-latest
outputs:
unique_runner_label: ${{steps.generator.outputs.runner_label}}
steps:
- name: Delete all runners
uses: actions/github-script@v3
with:
debug: true
script: |
var runnersResp = await github.actions.listSelfHostedRunnersForRepo({
owner: 'actions',
repo: 'runner',
per_page: '100'
});
for(var i=0; i<runnersResp.data.total_count; i++){
core.debug(JSON.stringify(runnersResp.data.runners[i]))
await github.actions.deleteSelfHostedRunnerFromRepo({
owner: 'actions',
repo: 'runner',
runner_id: runnersResp.data.runners[i].id
});
}
github-token: ${{secrets.PAT}}
- name: Generate Unique Runner label
id: generator
run: |
label=$(openssl rand -hex 16)
echo ::set-output name=runner_label::$label
build:
name: Build runner packages 🏗 📦
strategy:
matrix:
runtime: [ linux-x64, linux-arm64, linux-arm, win-x64, osx-x64 ]
include:
- runtime: linux-x64
os: ubuntu-latest
devScript: ./dev.sh
- runtime: linux-arm64
os: ubuntu-latest
devScript: ./dev.sh
- runtime: linux-arm
os: ubuntu-latest
devScript: ./dev.sh
- runtime: osx-x64
os: macOS-latest
devScript: ./dev.sh
- runtime: win-x64
os: windows-latest
devScript: ./dev
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v1
# Build runner layout
- name: Build & Layout Release
run: |
${{ matrix.devScript }} layout Release ${{ matrix.runtime }}
working-directory: src
# Create runner package tar.gz/zip
- name: Package Release
run: |
${{ matrix.devScript }} package Release ${{ matrix.runtime }}
working-directory: src
# Upload runner package tar.gz/zip as artifact
- name: Publish Artifact
uses: actions/upload-artifact@v1
with:
name: runner-package-${{ matrix.runtime }}
path: _package
dispatch_workflow:
name: Dispatch workflow to runners 🚨
needs: [init, build]
runs-on: ubuntu-latest
steps:
- name: Dispatch workflow
timeout-minutes: 10
uses: actions/github-script@v3
with:
debug: true
script: |
function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); }
async function dispatchWorkflow(runner) {
await github.actions.createWorkflowDispatch({
owner: 'actions',
repo: 'runner',
workflow_id: 'runner-basic-e2e-test-case.yml',
ref: 'main',
inputs: {target_runner: runner}
});
}
var runWin64 = false, runLinux64 = false, runOsx64 = false, runLinuxARM64 = false;
while (true) {
core.info(`------------- Waiting for runners to be configured --------------`)
await sleep(10000);
var runnersResp = await github.actions.listSelfHostedRunnersForRepo({owner: 'actions', repo: 'runner', per_page: '100'});
for (var i = 0; i < runnersResp.data.total_count; i++) {
core.debug(JSON.stringify(runnersResp.data.runners[i]))
var labels = runnersResp.data.runners[i].labels;
for (var j = 0; j < labels.length; j++) {
core.debug(`Comparing: ${labels[j].name} to win-x64/linux-x64/osx-x64/linux-arm64-${{ needs.init.outputs.unique_runner_label }}`)
if (labels[j].name == 'win-x64-${{needs.init.outputs.unique_runner_label}}' && runWin64 == false) {
core.info(`------------------- Windows runner is configured, queue Windows Run -------------------------`)
runWin64 = true;
await dispatchWorkflow('win-x64-${{needs.init.outputs.unique_runner_label}}');
break;
} else if (labels[j].name == 'linux-x64-${{needs.init.outputs.unique_runner_label}}' && runLinux64 == false) {
core.info(`------------------- Linux runner is configured, queue Linux Run -------------------------`)
runLinux64 = true;
await dispatchWorkflow('linux-x64-${{needs.init.outputs.unique_runner_label}}');
break;
} else if (labels[j].name == 'osx-x64-${{needs.init.outputs.unique_runner_label}}' && runOsx64 == false) {
core.info(`------------------- macOS runner is configured, queue macOS Run -------------------------`)
runOsx64 = true;
await dispatchWorkflow('osx-x64-${{needs.init.outputs.unique_runner_label}}');
break;
} else if (labels[j].name == 'linux-arm64-${{needs.init.outputs.unique_runner_label}}' && runLinuxARM64 == false) {
core.info(`------------------- Linux ARM64 runner is configured, queue Linux ARM64 Run-------------------------`)
runLinuxARM64 = true;
await dispatchWorkflow('linux-arm64-${{needs.init.outputs.unique_runner_label}}');
break;
}
}
}
if (runWin64 && runLinux64 && runOsx64 && runLinuxARM64) {
core.info(`--------------------- ALL runner are running jobs --------------------------`)
break;
} else {
core.info(`---------- Windows running: ${runWin64} -- Linux running: ${runLinux64} -- macOS running: ${runOsx64} -- Linux ARM64 running: ${runLinuxARM64} -----------`)
}
}
github-token: ${{secrets.PAT}}
LinuxE2E:
needs: [build, init]
runs-on: ubuntu-latest
steps:
- name: Download Runner
uses: actions/download-artifact@v2
with:
name: runner-package-linux-x64
- name: Unzip Runner Package
run: |
tar -xzf *.tar.gz
- name: Configure Runner
env:
unique_runner_name: linux-x64-${{needs.init.outputs.unique_runner_label}}
run: |
./config.sh --url ${{github.event.repository.html_url}} --unattended --name $unique_runner_name --pat ${{secrets.PAT}} --labels $unique_runner_name --replace
- name: Start Runner and Wait for Job
timeout-minutes: 5
run: |
./run.sh --once
- name: Remove Runner
if: always()
continue-on-error: true
run: |
./config.sh remove --pat ${{secrets.PAT}}
- name: Upload Runner Logs
if: always()
uses: actions/upload-artifact@v2
with:
name: linux_x64_logs
path: _diag
macOSE2E:
needs: [build, init]
runs-on: macos-latest
steps:
- name: Download Runner
uses: actions/download-artifact@v2
with:
name: runner-package-osx-x64
- name: Unzip Runner Package
run: |
tar -xzf *.tar.gz
- name: Configure Runner
env:
unique_runner_name: osx-x64-${{needs.init.outputs.unique_runner_label}}
run: |
./config.sh --url ${{github.event.repository.html_url}} --unattended --name $unique_runner_name --pat ${{secrets.PAT}} --labels $unique_runner_name --replace
- name: Start Runner and Wait for Job
timeout-minutes: 5
run: |
./run.sh --once
- name: Remove Runner
if: always()
continue-on-error: true
run: |
./config.sh remove --pat ${{secrets.PAT}}
- name: Upload Runner Logs
if: always()
uses: actions/upload-artifact@v2
with:
name: osx_x64_logs
path: _diag
ARM64E2E:
needs: [build, init]
runs-on: ubuntu-latest
steps:
- name: Download Runner
uses: actions/download-artifact@v2
with:
name: runner-package-linux-arm64
- name: Unzip Runner Package
run: |
tar -xzf *.tar.gz
- name: Prepare QEMU
run: |
docker run --rm --privileged multiarch/qemu-user-static:register --reset
- name: Configure Runner
uses: docker://multiarch/ubuntu-core:arm64-bionic
with:
args: 'bash -c "apt-get update && apt-get install -y curl && ./bin/installdependencies.sh && ./config.sh --unattended --name $unique_runner_name --url ${{github.event.repository.html_url}} --pat ${{secrets.PAT}} --labels $unique_runner_name --replace"'
env:
RUNNER_ALLOW_RUNASROOT: 1
unique_runner_name: linux-arm64-${{needs.init.outputs.unique_runner_label}}
- name: Start Runner and Wait for Job
timeout-minutes: 5
uses: docker://multiarch/ubuntu-core:arm64-bionic
with:
args: 'bash -c "apt-get update && apt-get install -y curl git && ./bin/installdependencies.sh && ./run.sh --once"'
env:
RUNNER_ALLOW_RUNASROOT: 1
- name: Remove Runner
if: always()
continue-on-error: true
uses: docker://multiarch/ubuntu-core:arm64-bionic
with:
args: 'bash -c "apt-get update && apt-get install -y curl && ./bin/installdependencies.sh && ./config.sh remove --pat ${{secrets.PAT}}"'
env:
RUNNER_ALLOW_RUNASROOT: 1
- name: Upload Runner Logs
if: always()
uses: actions/upload-artifact@v2
with:
name: linux_arm64_logs
path: _diag
WindowsE2E:
needs: [build, init]
runs-on: windows-latest
steps:
- name: Download Runner
uses: actions/download-artifact@v2
with:
name: runner-package-win-x64
- name: Unzip Runner Package
run: |
Get-ChildItem *.zip | Expand-Archive -DestinationPath $PWD
- name: Configure Runner
shell: cmd
run: |
config.cmd --unattended --url ${{github.event.repository.html_url}} --name %unique_runner_name% --pat ${{secrets.PAT}} --labels %unique_runner_name% --replace
env:
unique_runner_name: win-x64-${{needs.init.outputs.unique_runner_label}}
- name: Start Runner and Wait for Job
shell: cmd
timeout-minutes: 5
run: |
run.cmd --once
- name: Remove Runner
shell: cmd
if: always()
continue-on-error: true
run: |
config.cmd remove --pat ${{secrets.PAT}}
- name: Upload Runner Logs
if: always()
uses: actions/upload-artifact@v2
with:
name: win_x64_logs
path: _diag
check:
name: Check runner logs 🕵️‍♂️
needs: [WindowsE2E, LinuxE2E, macOSE2E, ARM64E2E]
runs-on: ubuntu-latest
steps:
- name: Download Linux Runner Logs
uses: actions/download-artifact@v2
with:
name: linux_x64_logs
path: linux_x64_logs
- name: Download macOS Runner Logs
uses: actions/download-artifact@v2
with:
name: osx_x64_logs
path: osx_x64_logs
- name: Download Linux ARM64 Runner Logs
uses: actions/download-artifact@v2
with:
name: linux_arm64_logs
path: linux_arm64_logs
- name: Download Windows Runner Logs
uses: actions/download-artifact@v2
with:
name: win_x64_logs
path: win_x64_logs
- name: Check Runner Logs
run: |
function failed()
{
local error=${1:-Undefined error}
echo "Failed: $error" >&2
exit 1
}
grep -R "completed with result: Succeeded" ./win_x64_logs || failed "Windows Runner fail to run the job, please check logs"
grep -R "completed with result: Succeeded" ./linux_x64_logs || failed "Linux Runner fail to run the job, please check logs"
grep -R "completed with result: Succeeded" ./osx_x64_logs || failed "macOS Runner fail to run the job, please check logs"
grep -R "completed with result: Succeeded" ./linux_arm64_logs || failed "Linux ARM64 Runner fail to run the job, please check logs"

View File

@@ -1,13 +1,14 @@
name: Runner CD name: Runner CD
on: on:
workflow_dispatch:
push: push:
paths: paths:
- releaseVersion - releaseVersion
jobs: jobs:
check: check:
if: startsWith(github.ref, 'refs/heads/releases/') || github.ref == 'refs/heads/master' if: startsWith(github.ref, 'refs/heads/releases/') || github.ref == 'refs/heads/main'
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2

View File

@@ -0,0 +1,31 @@
name: Runner Basics Test Case
on:
workflow_dispatch:
inputs:
target_runner:
description: 'Self-hosted runner will run the job'
required: true
jobs:
test:
runs-on:
- self-hosted
- ${{github.event.inputs.target_runner}}
name: Runner Basic Test 🛠
steps:
- uses: actions/checkout@v2
- name: Run a one-line script
run: echo Hello, world!
- name: Run a multi-line script
shell: bash
run: |
printenv|sort
cat $GITHUB_EVENT_PATH
- name: Validate GitHub Context
shell: bash
run: |
declare -a context_vars=("GITHUB_ACTION" "GITHUB_ACTIONS" "GITHUB_REPOSITORY" "GITHUB_WORKSPACE" "GITHUB_SHA" "GITHUB_RUN_ID" "GITHUB_RUN_NUMBER")
for var in ${context_vars[@]};
do [ -z "${!var}" ] && echo "##[error]$var not found" && exit 1 || echo "$var: ${!var}"; done

1
CODEOWNERS Normal file
View File

@@ -0,0 +1 @@
* @actions/actions-runtime

View File

@@ -5,8 +5,9 @@
# GitHub Actions Runner # GitHub Actions Runner
[![Actions Status](https://github.com/actions/runner/workflows/Runner%20CI/badge.svg)](https://github.com/actions/runner/actions) [![Actions Status](https://github.com/actions/runner/workflows/Runner%20CI/badge.svg)](https://github.com/actions/runner/actions)
[![Runner E2E Test](https://github.com/actions/runner/workflows/Runner%20E2E%20Test/badge.svg)](https://github.com/actions/runner/actions)
The runner is the application that runs a job from a GitHub Actions workflow. The runner can run on the [hosted machine pools](https://github.com/actions/virtual-environments) or run on [self-hosted environments](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/about-self-hosted-runners). The runner is the application that runs a job from a GitHub Actions workflow. It is used by GitHub Actions in the [hosted virtual environments](https://github.com/actions/virtual-environments), or you can [self-host the runner](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/about-self-hosted-runners) in your own environment.
## Get Started ## Get Started

View File

@@ -22,7 +22,7 @@ These are described in detail below:
- http://proxy.com - http://proxy.com
- http://127.0.0.1:8080 - http://127.0.0.1:8080
- http://user:password@proxy.com - http://user:password@proxy.com
- `no_proxy` a comma seperated list of hosts that should not use the proxy. An optional port may be specified - `no_proxy` a comma separated list of hosts that should not use the proxy. An optional port may be specified
- `google.com` - `google.com`
- `yahoo.com:443` - `yahoo.com:443`
- `google.com,bing.com` - `google.com,bing.com`
@@ -31,9 +31,9 @@ We won't use `http_proxy` for https traffic when `https_proxy` is not set, this
Otherwise action authors and workflow users need to adjust to differences between the runner proxy convention, and tools used by their actions and scripts. Otherwise action authors and workflow users need to adjust to differences between the runner proxy convention, and tools used by their actions and scripts.
Example: Example:
Customer set `http_proxy=http://127.0.0.1:8888` and configure the runner against `https://github.com/owner/repo`, with the `https_proxy` -> `http_proxy` fallback, the runner will connect to server without any problem. However, if user runs `git push` to `https://github.com/owner/repo`, `git` won't use the proxy since it require `https_proxy` to be set for any https traffic. Customer set `http_proxy=http://127.0.0.1:8888` and configure the runner against `https://github.com/owner/repo`, with the `https_proxy` -> `http_proxy` fallback, the runner will connect to the server without any problem. However, if a user runs `git push` to `https://github.com/owner/repo`, `git` won't use the proxy since it requires `https_proxy` to be set for any https traffic.
> `golang`, `node.js` and other dev tools from the linux community use `http_proxy` for both http and https traffic base on my research. > `golang`, `node.js` and other dev tools from the linux community use `http_proxy` for both http and https traffic based on my research.
A majority of our users are using Linux where these variables are commonly required to be set by various programs. By reading these values, we simplify the process for self hosted runners to set up proxy, and expose it in a way users are already familiar with. A majority of our users are using Linux where these variables are commonly required to be set by various programs. By reading these values, we simplify the process for self hosted runners to set up proxy, and expose it in a way users are already familiar with.
@@ -43,7 +43,7 @@ We will support the lowercase and uppercase variants, with lowercase taking prio
### No Proxy Format ### No Proxy Format
While exact implementations are different per application on handle `no_proxy` env, most applications accept a comma separated list of hosts. Some accept wildcard characters (*). We are going to do exact case-insentive matches, and not support wildcards at this time. While exact implementations are different per application on handle `no_proxy` env, most applications accept a comma separated list of hosts. Some accept wildcard characters (*). We are going to do exact case-insensitive matches, and not support wildcards at this time.
For example: For example:
- example.com will match example.com, foo.example.com, foo.bar.example.com - example.com will match example.com, foo.example.com, foo.bar.example.com
- foo.example.com will match bar.foo.example.com and foo.example.com - foo.example.com will match bar.foo.example.com and foo.example.com
@@ -57,5 +57,5 @@ We will not support IP addresses for `no_proxy`, only hostnames.
3. The runner will read from the environmental variables during config and runtime and use the provided proxy if it exists 3. The runner will read from the environmental variables during config and runtime and use the provided proxy if it exists
4. Users may need to pass these environmental variables into other applications if they do not natively take these variables 4. Users may need to pass these environmental variables into other applications if they do not natively take these variables
5. Action authors may need to update their workflows to react to the these environment variables 5. Action authors may need to update their workflows to react to the these environment variables
6. We will document the way of setting environmental variables for runners using the environmental variables and how the runner uses them 6. We will document the way of setting environmental variables for runners using the environment variables and how the runner uses them
7. Like all other secrets, users will be able to relatively easily figure out proxy password if they can modify a workflow file running on a self hosted machine 7. Like all other secrets, users will be able to relatively easily figure out proxy password if they can modify a workflow file running on a self hosted machine

View File

@@ -34,7 +34,7 @@ A way out for rare cases where scoping is a problem.
`##[remove-matcher]owner` `##[remove-matcher]owner`
For the this to be usable, the `owner` needs to be discoverable. Therefore, debug print the owner on registration. For this to be usable, the `owner` needs to be discoverable. Therefore, debug print the owner on registration.
### Single line matcher ### Single line matcher
@@ -184,7 +184,7 @@ Solving this problem means:
- Use the `github.workspace` (where the repo is cloned on disk) - Use the `github.workspace` (where the repo is cloned on disk)
- Match against a repository to determine the relative path within the repo - Match against a repository to determine the relative path within the repo
This is a place where we diverge from VSCode. VSCode task configuration are specific to the local workspace (workspace root is known or can be specified). We're solving a more generic problem, so we need more information - specifically the `fromPath` property - in order to accurately root the path. This is a place where we diverge from VSCode. VSCode task configurations are specific to the local workspace (workspace root is known or can be specified). We're solving a more generic problem, so we need more information - specifically the `fromPath` property - in order to accurately root the path.
In order to avoid creating inaccurate hyperlinks on the error issues, the agent will verify the file exists and is in the main repository. Otherwise omit the file property from the error issue and debug trace what happened. In order to avoid creating inaccurate hyperlinks on the error issues, the agent will verify the file exists and is in the main repository. Otherwise omit the file property from the error issue and debug trace what happened.
@@ -203,7 +203,7 @@ Problem matchers are unable to interpret severity strings other than `warning` a
However some tools indicate error/warning in different ways. For example `flake8` uses codes like `E100`, `W200`, and `F300` (error, warning, fatal, respectively). However some tools indicate error/warning in different ways. For example `flake8` uses codes like `E100`, `W200`, and `F300` (error, warning, fatal, respectively).
Therefore, allow a property `severity`, sibling to `owner`, which identifies the default severity for the problem matcher. This allows two problem matchers are registered - one for warnings and one for errors. Therefore, allow a property `severity`, sibling to `owner`, which identifies the default severity for the problem matcher. This allows two problem matchers to be registered - one for warnings and one for errors.
For example, given the following `flake8` output: For example, given the following `flake8` output:

View File

@@ -84,7 +84,7 @@ powershell/pwsh
- Users can always opt out by not using the builtins, and providing a shell option like: `pwsh -File {0}`, or `powershell -Command "& '{0}'"`, depending on need - Users can always opt out by not using the builtins, and providing a shell option like: `pwsh -File {0}`, or `powershell -Command "& '{0}'"`, depending on need
cmd cmd
- There doesnt seem to be a way to fully opt in to fail-fast behavior other than writing your script to check each error code and respond accordingly, so we cant actually provide that behavior by default, it will be completely up to the user to write this behavior into their script - There doesn't seem to be a way to fully opt in to fail-fast behavior other than writing your script to check each error code and respond accordingly, so we can't actually provide that behavior by default, it will be completely up to the user to write this behavior into their script
- cmd.exe will exit (return the error code to the runner) with the errorlevel of the last program it executed. This is internally consistent with the previous default behavior (sh, pwsh) and is the cmd.exe default, so we keep that behavior - cmd.exe will exit (return the error code to the runner) with the errorlevel of the last program it executed. This is internally consistent with the previous default behavior (sh, pwsh) and is the cmd.exe default, so we keep that behavior
## Consequences ## Consequences

View File

@@ -0,0 +1,378 @@
# ADR 0549: Composite Run Steps
**Date**: 2020-06-17
**Status**: Accepted
## Context
Customers want to be able to compose actions from actions (ex: https://github.com/actions/runner/issues/438)
An important step towards meeting this goal is to build in functionality for actions where users can simply execute any number of steps.
### Guiding Principles
We don't want the workflow author to need to know how the internal workings of the action work. Users shouldn't know the internal workings of the composite action (for example, `default.shell` and `default.workingDir` should not be inherited from the workflow file to the action file). When deciding how to design certain parts of composite run steps, we want to think one logical step from the consumer.
A composite action is treated as **one** individual job step (this is known as encapsulation).
## Decision
**In this ADR, we only support running multiple run steps in an Action.** In doing so, we build in support for mapping and flowing the inputs, outputs, and env variables (ex: All nested steps should have access to its parents' input variables and nested steps can overwrite the input variables).
### Composite Run Steps Features
This feature supports at the top action level:
- name
- description
- inputs
- runs
- outputs
This feature supports at the run step level:
- name
- id
- run
- env
- shell
- working-directory
This feature **does not support** at the run step level:
- timeout-minutes
- secrets
- conditionals (needs, if, etc.)
- continue-on-error
### Steps
Example `workflow.yml`
```yaml
jobs:
build:
runs-on: self-hosted
steps:
- id: step1
uses: actions/setup-python@v1
- id: step2
uses: actions/setup-node@v2
- uses: actions/checkout@v2
- uses: user/composite@v1
- name: workflow step 1
run: echo hello world 3
- name: workflow step 2
run: echo hello world 4
```
Example `user/composite/action.yml`
```yaml
runs:
using: "composite"
steps:
- run: pip install -r requirements.txt
shell: bash
- run: npm install
shell: bash
```
Example Output
```yaml
[npm installation output]
[pip requirements output]
echo hello world 3
echo hello world 4
```
We add a token called "composite" which allows our Runner code to process composite actions. By invoking "using: composite", our Runner code then processes the "steps" attribute, converts this template code to a list of steps, and finally runs each run step sequentially. If any step fails and there are no `if` conditions defined, the whole composite action job fails.
### Defaults
We will not support "defaults" in a composite action.
### Shell and Working-directory
For each run step in a composite action, the action author can set the `shell` and `working-directory` attributes for that step. The shell attribute is **required** for each run step because the action author does not know what the workflow author is using for the operating system so we need to explicitly prevent unknown behavior by making sure that each run step has an explicit shell **set by the action author.** On the other hand, `working-directory` is optional. Moreover, the composite action author can map in values from the `inputs` for it's `shell` and `working-directory` attributes at the step level for an action.
For example,
`action.yml`
```yaml
inputs:
shell_1:
description: 'Your name'
default: 'pwsh'
steps:
- run: echo 1
shell: ${{ inputs.shell_1 }}
```
Note, the workflow file and action file are treated as separate entities. **So, the workflow `defaults` will never change the `shell` and `working-directory` value in the run steps in a composite action.** Note, `defaults` in a workflow only apply to run steps not "uses" steps (steps that use an action).
### Running Local Scripts
Example 'workflow.yml':
```yaml
jobs:
build:
runs-on: self-hosted
steps:
- uses: user/composite@v1
```
Example `user/composite/action.yml`:
```yaml
runs:
using: "composite"
steps:
- run: chmod +x ${{ github.action_path }}/test/script2.sh
shell: bash
- run: chmod +x $GITHUB_ACTION_PATH/script.sh
shell: bash
- run: ${{ github.action_path }}/test/script2.sh
shell: bash
- run: $GITHUB_ACTION_PATH/script.sh
shell: bash
```
Where `user/composite` has the file structure:
```
.
+-- action.yml
+-- script.sh
+-- test
| +-- script2.sh
```
Users will be able to run scripts located in their action folder by first prepending the relative path and script name with `$GITHUB_ACTION_PATH` or `github.action_path` which contains the path in which the composite action is downloaded to and where those "files" live. Note, you'll have to use `chmod` before running each script if you do not git check in your script files into your github repo with the executable bit turned on.
### Inputs
Example `workflow.yml`:
```yaml
steps:
- id: foo
uses: user/composite@v1
with:
your_name: "Octocat"
```
Example `user/composite/action.yml`:
```yaml
inputs:
your_name:
description: 'Your name'
default: 'Ethan'
runs:
using: "composite"
steps:
- run: echo hello ${{ inputs.your_name }}
shell: bash
```
Example Output:
```
hello Octocat
```
Each input variable in the composite action is only viewable in its own scope.
### Outputs
Example `workflow.yml`:
```yaml
...
steps:
- id: foo
uses: user/composite@v1
- run: echo random-number ${{ steps.foo.outputs.random-number }}
shell: bash
```
Example `user/composite/action.yml`:
```yaml
outputs:
random-number:
description: "Random number"
value: ${{ steps.random-number-generator.outputs.random-id }}
runs:
using: "composite"
steps:
- id: random-number-generator
run: echo "::set-output name=random-id::$(echo $RANDOM)"
shell: bash
```
Example Output:
```
::set-output name=my-output::43243
random-number 43243
```
Each of the output variables from the composite action is viewable from the workflow file that uses the composite action. In other words, every child action output(s) is viewable only by its parent using dot notation (ex `steps.foo.outputs.random-number`).
Moreover, the output ids are only accessible within the scope where it was defined. Note that in the example above, in our `workflow.yml` file, it should not have access to output id (i.e. `random-id`). The reason why we are doing this is because we don't want to require the workflow author to know the internal workings of the composite action.
### Context
Similar to the workflow file, the composite action has access to the [same context objects](https://help.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#contexts) (ex: `github`, `env`, `strategy`).
### Environment
In the Composite Action, you'll only be able to use `::set-env::` to set environment variables just like you could with other actions.
### Secrets
**We will not support "Secrets" in a composite action for now. This functionality will be focused on in a future ADR.**
We'll pass the secrets from the composite action's parents (ex: the workflow file) to the composite action. Secrets can be created in the composite action with the secrets context. In the actions yaml, we'll automatically mask the secret.
### If Condition
** If and needs conditions will not be supported in the composite run steps feature. It will be supported later on in a new feature. **
Old reasoning:
Example `workflow.yml`:
```yaml
steps:
- run: exit 1
- uses: user/composite@v1 # <--- this will run, as it's marked as always runing
if: always()
```
Example `user/composite/action.yml`:
```yaml
runs:
using: "composite"
steps:
- run: echo "just succeeding"
shell: bash
- run: echo "I will run, as my current scope is succeeding"
shell: bash
if: success()
- run: exit 1
shell: bash
- run: echo "I will not run, as my current scope is now failing"
shell: bash
```
**We will not support "if Condition" in a composite action for now. This functionality will be focused on in a future ADR.**
See the paragraph below for a rudimentary approach (thank you to @cybojenix for the idea, example, and explanation for this approach):
The `if` statement in the parent (in the example above, this is the `workflow.yml`) shows whether or not we should run the composite action. So, our composite action will run since the `if` condition for running the composite action is `always()`.
**Note that the if condition on the parent does not propagate to the rest of its children though.**
In the child action (in this example, this is the `action.yml`), it starts with a clean slate (in other words, no imposing if conditions). Similar to the logic in the paragraph above, `echo "I will run, as my current scope is succeeding"` will run since the `if` condition checks if the previous steps **within this composite action** has not failed. `run: echo "I will not run, as my current scope is now failing"` will not run since the previous step resulted in an error and by default, the if expression is set to `success()` if the if condition is not set for a step.
What if a step has `cancelled()`? We do the opposite of our approach above if `cancelled()` is used for any of our composite run steps. We will cancel any step that has this condition if the workflow is cancelled at all.
### Timeout-minutes
Example `workflow.yml`:
```yaml
steps:
- id: bar
uses: user/test@v1
timeout-minutes: 50
```
Example `user/composite/action.yml`:
```yaml
runs:
using: "composite"
steps:
- id: foo1
run: echo test 1
timeout-minutes: 10
shell: bash
- id: foo2
run: echo test 2
shell: bash
- id: foo3
run: echo test 3
timeout-minutes: 10
shell: bash
```
**We will not support "timeout-minutes" in a composite action for now. This functionality will be focused on in a future ADR.**
A composite action in its entirety is a job. You can set both timeout-minutes for the whole composite action or its steps as long as the the sum of the `timeout-minutes` for each composite action step that has the attribute `timeout-minutes` is less than or equals to `timeout-minutes` for the composite action. There is no default timeout-minutes for each composite action step.
If the time taken for any of the steps in combination or individually exceed the whole composite action `timeout-minutes` attribute, the whole job will fail (1). If an individual step exceeds its own `timeout-minutes` attribute but the total time that has been used including this step is below the overall composite action `timeout-minutes`, the individual step will fail but the rest of the steps will run based on their own `timeout-minutes` attribute (they will still abide by condition (1) though).
For reference, in the example above, if the composite step `foo1` takes 11 minutes to run, that step will fail but the rest of the steps, `foo1` and `foo2`, will proceed as long as their total runtime with the previous failed `foo1` action is less than the composite action's `timeout-minutes` (50 minutes). If the composite step `foo2` takes 51 minutes to run, it will cause the whole composite action job to fail. I
The rationale behind this is that users can configure their steps with the `if` condition to conditionally set how steps rely on each other. Due to the additional capabilities that are offered with combining `timeout-minutes` and/or `if`, we wanted the `timeout-minutes` condition to be as dumb as possible and not effect other steps.
[Usage limits still apply](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions?query=if%28%29#usage-limits)
### Continue-on-error
Example `workflow.yml`:
```yaml
steps:
- run: exit 1
- id: bar
uses: user/test@v1
continue-on-error: false
- id: foo
run: echo "Hello World" <------- This step will not run
```
Example `user/composite/action.yml`:
```yaml
runs:
using: "composite"
steps:
- run: exit 1
continue-on-error: true
shell: bash
- run: echo "Hello World 2" <----- This step will run
shell: bash
```
**We will not support "continue-on-error" in a composite action for now. This functionality will be focused on in a future ADR.**
If any of the steps fail in the composite action and the `continue-on-error` is set to `false` for the whole composite action step in the workflow file, then the steps below it will run. On the flip side, if `continue-on-error` is set to `true` for the whole composite action step in the workflow file, the next job step will run.
For the composite action steps, it follows the same logic as above. In this example, `"Hello World 2"` will be outputted because the previous step has `continue-on-error` set to `true` although that previous step errored.
### Visualizing Composite Action in the GitHub Actions UI
We want all the composite action's steps to be condensed into the original composite action node.
Here is a visual represenation of the [first example](#Steps)
```yaml
| composite_action_node |
| echo hello world 1 |
| echo hello world 2 |
| echo hello world 3 |
| echo hello world 4 |
```
## Consequences
This ADR lays the framework for eventually supporting nested Composite Actions within Composite Actions. This ADR allows for users to run multiple run steps within a GitHub Composite Action with the support of inputs, outputs, environment, and context for use in any steps as well as the if, timeout-minutes, and the continue-on-error attributes for each Composite Action step.

44
docs/checks/actions.md Normal file
View File

@@ -0,0 +1,44 @@
# Actions Connection Check
## What is this check for?
Make sure the runner has access to actions service for GitHub.com or GitHub Enterprise Server
- For GitHub.com
- The runner needs to access https://api.github.com for downloading actions.
- The runner needs to access https://vstoken.actions.githubusercontent.com/_apis/.../ for requesting an access token.
- The runner needs to access https://pipelines.actions.githubusercontent.com/_apis/.../ for receiving workflow jobs.
- For GitHub Enterprise Server
- The runner needs to access https://myGHES.com/api/v3 for downloading actions.
- The runner needs to access https://myGHES.com/_services/vstoken/_apis/.../ for requesting an access token.
- The runner needs to access https://myGHES.com/_services/pipelines/_apis/.../ for receiving workflow jobs.
## What is checked?
- DNS lookup for api.github.com or myGHES.com using dotnet
- Ping api.github.com or myGHES.com using dotnet
- Make HTTP GET to https://api.github.com or https://myGHES.com/api/v3 using dotnet, check response headers contains `X-GitHub-Request-Id`
---
- DNS lookup for vstoken.actions.githubusercontent.com using dotnet
- Ping vstoken.actions.githubusercontent.com using dotnet
- Make HTTP GET to https://vstoken.actions.githubusercontent.com/_apis/health or https://myGHES.com/_services/vstoken/_apis/health using dotnet, check response headers contains `x-vss-e2eid`
---
- DNS lookup for pipelines.actions.githubusercontent.com using dotnet
- Ping pipelines.actions.githubusercontent.com using dotnet
- Make HTTP GET to https://pipelines.actions.githubusercontent.com/_apis/health or https://myGHES.com/_services/pipelines/_apis/health using dotnet, check response headers contains `x-vss-e2eid`
## How to fix the issue?
### 1. Check the common network issue
> Please check the [network doc](./network.md)
### 2. SSL certificate related issue
If you are seeing `System.Net.Http.HttpRequestException: The SSL connection could not be established, see inner exception.` in the log, it means the runner can't connect to Actions service due to SSL handshake failure.
> Please check the [SSL cert doc](./sslcert.md)
## Still not working?
Contact GitHub customer service or log an issue at https://github.com/actions/runner if you think it's a runner issue.

34
docs/checks/git.md Normal file
View File

@@ -0,0 +1,34 @@
# Git Connection Check
## What is this check for?
Make sure `git` can access GitHub.com or your GitHub Enterprise Server.
## What is checked?
The test is done by executing
```bash
# For GitHub.com
git ls-remote --exit-code https://github.com/actions/checkout HEAD
# For GitHub Enterprise Server
git ls-remote --exit-code https://ghes.me/actions/checkout HEAD
```
The test also set environment variable `GIT_TRACE=1` and `GIT_CURL_VERBOSE=1` before running `git ls-remote`, this will make `git` to produce debug log for better debug any potential issues.
## How to fix the issue?
### 1. Check the common network issue
> Please check the [network doc](./network.md)
### 2. SSL certificate related issue
If you are seeing `SSL Certificate problem:` in the log, it means the `git` can't connect to the GitHub server due to SSL handshake failure.
> Please check the [SSL cert doc](./sslcert.md)
## Still not working?
Contact GitHub customer service or log an issue at https://github.com/actions/runner if you think it's a runner issue.

26
docs/checks/internet.md Normal file
View File

@@ -0,0 +1,26 @@
# Internet Connection Check
## What is this check for?
Make sure the runner has access to https://api.github.com
The runner needs to access https://api.github.com to download any actions from the marketplace.
Even the runner is configured to GitHub Enterprise Server, the runner can still download actions from GitHub.com with [GitHub Connect](https://docs.github.com/en/enterprise-server@2.22/admin/github-actions/enabling-automatic-access-to-githubcom-actions-using-github-connect)
## What is checked?
- DNS lookup for api.github.com using dotnet
- Ping api.github.com using dotnet
- Make HTTP GET to https://api.github.com using dotnet, check response headers contains `X-GitHub-Request-Id`
## How to fix the issue?
### 1. Check the common network issue
> Please check the [network doc](./network.md)
## Still not working?
Contact GitHub customer service or log an issue at https://github.com/actions/runner if you think it's a runner issue.

29
docs/checks/network.md Normal file
View File

@@ -0,0 +1,29 @@
## Common Network Related Issues
### Common things that can cause the runner to not working properly
- Bug in the runner or the dotnet framework that causes actions runner can't make Http request in a certain network environment.
- Proxy/Firewall block certain HTTP method, like it block all POST and PUT calls which the runner will use to upload logs.
- Proxy/Firewall only allows requests with certain user-agent to pass through and the actions runner user-agent is not in the allow list.
- Proxy try to decrypt and exam HTTPS traffic for security purpose but cause the actions-runner to fail to finish SSL handshake due to the lack of trusting proxy's CA.
- Firewall rules that block action runner from accessing certain hosts, ex: `*.github.com`, `*.actions.githubusercontent.com`, etc.
### Identify and solve these problems
The key is to figure out where is the problem, the network environment, or the actions runner?
Use a 3rd party tool to make the same requests as the runner did would be a good start point.
- Use `nslookup` to check DNS
- Use `ping` to check Ping
- Use `curl -v` to check the network stack, good for verifying default certificate/proxy settings.
- Use `Invoke-WebRequest` from `pwsh` (`PowerShell Core`) to check the dotnet network stack, good for verifying bugs in the dotnet framework.
If the 3rd party tool is also experiencing the same error as the runner does, then you might want to contact your network administrator for help.
Otherwise, contact GitHub customer support or log an issue at https://github.com/actions/runner

30
docs/checks/nodejs.md Normal file
View File

@@ -0,0 +1,30 @@
# Node.js Connection Check
## What is this check for?
Make sure the built-in node.js has access to GitHub.com or GitHub Enterprise Server.
The runner carries it's own copy of node.js executable under `<runner_root>/externals/node12/`.
All javascript base Actions will get executed by the built-in `node` at `<runner_root>/externals/node12/`.
> Not the `node` from `$PATH`
## What is checked?
- Make HTTPS GET to https://api.github.com or https://myGHES.com/api/v3 using node.js, make sure it gets 200 response code.
## How to fix the issue?
### 1. Check the common network issue
> Please check the [network doc](./network.md)
### 2. SSL certificate related issue
If you are seeing `Https request failed due to SSL cert issue` in the log, it means the `node.js` can't connect to the GitHub server due to SSL handshake failure.
> Please check the [SSL cert doc](./sslcert.md)
## Still not working?
Contact GitHub customer service or log an issue at https://github.com/actions/runner if you think it's a runner issue.

89
docs/checks/sslcert.md Normal file
View File

@@ -0,0 +1,89 @@
## SSL Certificate Related Issues
You might run into an SSL certificate error when your GitHub Enterprise Server is using a self-signed SSL server certificate or a web proxy within your network is decrypting HTTPS traffic for a security audit.
As long as your certificate is generated properly, most of the issues should be fixed after your trust the certificate properly on the runner machine.
> Different OS might have extra requirements on SSL certificate,
> Ex: macOS requires `ExtendedKeyUsage` https://support.apple.com/en-us/HT210176
### Don't skip SSL cert validation
> !!! DO NOT SKIP SSL CERT VALIDATION !!!
> !!! IT IS A BAD SECURITY PRACTICE !!!
### Download SSL certificate chain
Depends on how your SSL server certificate gets configured, you might need to download the whole certificate chain from a machine that has trusted the SSL certificate's CA.
- Approach 1: Download certificate chain using a browser (Chrome, Firefox, IT), you can google for more example, [here is what I found](https://medium.com/@menakajain/export-download-ssl-certificate-from-server-site-url-bcfc41ea46a2)
- Approach 2: Download certificate chain using OpenSSL, you can google for more example, [here is what I found](https://superuser.com/a/176721)
- Approach 3: Ask your network administrator or the owner of the CA certificate to send you a copy of it
### Trust CA certificate for the Runner
The actions runner is a dotnet core application which will follow how dotnet load SSL CA certificates on each OS.
You can get full details documentation at [here](https://docs.microsoft.com/en-us/dotnet/standard/security/cross-platform-cryptography#x509store)
In short:
- Windows: Load from Windows certificate store.
- Linux: Load from OpenSSL CA cert bundle.
- macOS: Load from macOS KeyChain.
To let the runner trusts your CA certificate, you will need to:
1. Save your SSL certificate chain which includes the root CA and all intermediate CAs into a `.pem` file.
2. Use `OpenSSL` to convert `.pem` file to a proper format for different OS, here is some [doc with sample commands](https://www.sslshopper.com/ssl-converter.html)
3. Trust CA on different OS:
- Windows: https://docs.microsoft.com/en-us/skype-sdk/sdn/articles/installing-the-trusted-root-certificate
- macOS: ![trust ca cert](./../res/macOStrustCA.gif)
- Linux: Refer to the distribution documentation
1. RedHat: https://www.redhat.com/sysadmin/ca-certificates-cli
2. Ubuntu: http://manpages.ubuntu.com/manpages/focal/man8/update-ca-certificates.8.html
3. Google search: "trust ca certificate on [linux distribution]"
4. If all approaches failed, set environment variable `SSL_CERT_FILE` to the CA bundle `.pem` file we get.
> To verity cert gets installed properly on Linux, you can try use `curl -v https://sitewithsslissue.com` and `pwsh -Command \"Invoke-WebRequest -Uri https://sitewithsslissue.com\"`
### Trust CA certificate for Git CLI
Git uses various CA bundle file depends on your operation system.
- Git packaged the CA bundle file within the Git installation on Windows
- Git use OpenSSL certificate CA bundle file on Linux and macOS
You can check where Git check CA file by running:
```bash
export GIT_CURL_VERBOSE=1
git ls-remote https://github.com/actions/runner HEAD
```
You should see something like:
```
* Couldn't find host github.com in the .netrc file; using defaults
* Trying 140.82.114.4...
* TCP_NODELAY set
* Connected to github.com (140.82.114.4) port 443 (#0)
* ALPN, offering h2
* ALPN, offering http/1.1
* successfully set certificate verify locations:
* CAfile: /etc/ssl/cert.pem
CApath: none
* SSL connection using TLSv1.2 / ECDHE-RSA-AES128-GCM-SHA256
```
This tells me `/etc/ssl/cert.pem` is where it read trusted CA certificates.
To let Git trusts your CA certificate, you will need to:
1. Save your SSL certificate chain which includes the root CA and all intermediate CAs into a `.pem` file.
2. Set `http.sslCAInfo` Git config or `GIT_SSL_CAINFO` environment variable to the full path of the `.pem` file [Git Doc](https://git-scm.com/docs/git-config#Documentation/git-config.txt-httpsslCAInfo)
> I would recommend using `http.sslCAInfo` since it can be scope to certain hosts that need the extra trusted CA.
> Ex: `git config --global http.https://myghes.com/.sslCAInfo /extra/ca/cert.pem`
> This will make Git use the `/extra/ca/cert.pem` only when communicates with `https://myghes.com` and keep using the default CA bundle with others.
### Trust CA certificate for Node.js
Node.js has compiled a snapshot of the Mozilla CA store that is fixed at each version of Node.js' release time.
To let Node.js trusts your CA certificate, you will need to:
1. Save your SSL certificate chain which includes the root CA and all intermediate CAs into a `.pem` file.
2. Set environment variable `NODE_EXTRA_CA_CERTS` which point to the file. ex: `export NODE_EXTRA_CA_CERTS=/full/path/to/cacert.pem` or `set NODE_EXTRA_CA_CERTS=C:\full\path\to\cacert.pem`

View File

@@ -14,7 +14,7 @@ Issues in this repository should be for the runner application. Note that the V
We ask that before significant effort is put into code changes, that we have agreement on taking the change before time is invested in code changes. We ask that before significant effort is put into code changes, that we have agreement on taking the change before time is invested in code changes.
1. Create a feature request. Once agreed we will take the enhancment 1. Create a feature request. Once agreed we will take the enhancement
2. Create an ADR to agree on the details of the change. 2. Create an ADR to agree on the details of the change.
An ADR is an Architectural Decision Record. This allows consensus on the direction forward and also serves as a record of the change and motivation. [Read more here](adrs/README.md) An ADR is an Architectural Decision Record. This allows consensus on the direction forward and also serves as a record of the change and motivation. [Read more here](adrs/README.md)
@@ -23,7 +23,7 @@ An ADR is an Architectural Decision Record. This allows consensus on the direct
### Required Dev Dependencies ### Required Dev Dependencies
![Win](res/win_sm.png) Git for Windows [Install Here](https://git-scm.com/downloads) (needed for dev sh script) ![Win](res/win_sm.png) ![*nix](res/linux_sm.png) Git for Windows and Linux [Install Here](https://git-scm.com/downloads) (needed for dev sh script)
### To Build, Test, Layout ### To Build, Test, Layout
@@ -43,6 +43,7 @@ Sample developer flow:
```bash ```bash
git clone https://github.com/actions/runner git clone https://github.com/actions/runner
cd runner
cd ./src cd ./src
./dev.(sh/cmd) layout # the runner that built from source is in {root}/_layout ./dev.(sh/cmd) layout # the runner that built from source is in {root}/_layout
<make code changes> <make code changes>
@@ -50,10 +51,23 @@ cd ./src
./dev.(sh/cmd) test # run all unit tests before git commit/push ./dev.(sh/cmd) test # run all unit tests before git commit/push
``` ```
View logs:
```bash
cd runner/_layout/_diag
ls
cat (Runner/Worker)_TIMESTAMP.log # view your log file
```
Run Runner:
```bash
cd runner/_layout
./run.sh # run your custom runner
```
### Editors ### Editors
[Using Visual Studio Code](https://code.visualstudio.com/) [Using Visual Studio Code](https://code.visualstudio.com/)
[Using Visual Studio 2019](https://www.visualstudio.com/vs/) [Using Visual Studio](https://code.visualstudio.com/docs)
### Styling ### Styling

BIN
docs/res/macOStrustCA.gif Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 MiB

View File

@@ -15,16 +15,16 @@ x64
- openSUSE 15+ - openSUSE 15+
- SUSE Enterprise Linux (SLES) 12 SP2+ - SUSE Enterprise Linux (SLES) 12 SP2+
## Install .Net Core 3.x Linux Dependencies ## Install .Net Core 5 Linux Dependencies
The `./config.sh` will check .Net Core 3.x dependencies during runner configuration. The `./config.sh` will check .Net Core 5 dependencies during runner configuration.
You might see something like this which indicate a dependency's missing. You might see something like this which indicate a dependency's missing.
```bash ```bash
./config.sh ./config.sh
libunwind.so.8 => not found libunwind.so.8 => not found
libunwind-x86_64.so.8 => not found libunwind-x86_64.so.8 => not found
Dependencies is missing for Dotnet Core 3.0 Dependencies is missing for Dotnet 5
Execute ./bin/installdependencies.sh to install any missing Dotnet Core 3.0 dependencies. Execute ./bin/installdependencies.sh to install any missing Dotnet 5 dependencies.
``` ```
You can easily correct the problem by executing `./bin/installdependencies.sh`. You can easily correct the problem by executing `./bin/installdependencies.sh`.
The `installdependencies.sh` script should install all required dependencies on all supported Linux versions The `installdependencies.sh` script should install all required dependencies on all supported Linux versions

View File

@@ -1,20 +1,11 @@
## Features ## Features
- Sample scripts to automate scalable runners (#427)
- Raise warning when action input does not match action.yml. (#429)
- Add secret masker for trimming double quotes. (#440)
- Use the API_URL and munge action URLs for GHES (#437 #469)
- Help trace worker crash in Kusto. (#450)
- update checkout@v1 for GHES (#470)
## Bugs ## Bugs
- Print node version in debug instead of output. (#433) - Downgrade runner to .NET 3 to address an issue with broken pipes in Ubuntu (#928)
- Better error when runner removed from service. (#441) - Fixed an issue where FIPS Cryptography broke back-compat scenarios (#928)
- Add help info for '--labels' config option (#472)
- Sps/token migration fix, job.status/steps.outcome/steps.conclusion case match with GitHub check suites conclusion. (#462)
- Docker build using -f instead of implied default (#471)
## Misc ## Misc
- Make release notes code blocks copy-paste-able (#430) - Updated dotnet install scripts (#928)
- Fix spelling of RHEL and CentOS. (#436)
- Add CodeQL Analysis workflow (#459)
## Windows x64 ## Windows x64
We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows.

View File

@@ -1 +1 @@
<Update to ./src/runnerversion when creating release> 2.276.1

View File

@@ -12,12 +12,13 @@ set -e
# #
# Usage: # Usage:
# export RUNNER_CFG_PAT=<yourPAT> # export RUNNER_CFG_PAT=<yourPAT>
# ./create-latest-svc scope [ghe_domain] [name] [user] # ./create-latest-svc scope [ghe_domain] [name] [user] [labels]
# #
# scope required repo (:owner/:repo) or org (:organization) # scope required repo (:owner/:repo) or org (:organization)
# ghe_domain optional the fully qualified domain name of your GitHub Enterprise Server deployment # ghe_domain optional the fully qualified domain name of your GitHub Enterprise Server deployment
# name optional defaults to hostname # name optional defaults to hostname
# user optional user svc will run as. defaults to current # user optional user svc will run as. defaults to current
# labels optional list of labels (split by comma) applied on the runner
# #
# Notes: # Notes:
# PATS over envvars are more secure # PATS over envvars are more secure
@@ -30,6 +31,7 @@ runner_scope=${1}
ghe_hostname=${2} ghe_hostname=${2}
runner_name=${3:-$(hostname)} runner_name=${3:-$(hostname)}
svc_user=${4:-$USER} svc_user=${4:-$USER}
labels=${5}
echo "Configuring runner @ ${runner_scope}" echo "Configuring runner @ ${runner_scope}"
sudo echo sudo echo
@@ -81,7 +83,7 @@ fi
export RUNNER_TOKEN=$(curl -s -X POST ${base_api_url}/${orgs_or_repos}/${runner_scope}/actions/runners/registration-token -H "accept: application/vnd.github.everest-preview+json" -H "authorization: token ${RUNNER_CFG_PAT}" | jq -r '.token') export RUNNER_TOKEN=$(curl -s -X POST ${base_api_url}/${orgs_or_repos}/${runner_scope}/actions/runners/registration-token -H "accept: application/vnd.github.everest-preview+json" -H "authorization: token ${RUNNER_CFG_PAT}" | jq -r '.token')
if [ -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi if [ "null" == "$RUNNER_TOKEN" -o -z "$RUNNER_TOKEN" ]; then fatal "Failed to get a token"; fi
#--------------------------------------- #---------------------------------------
# Download latest released and extract # Download latest released and extract
@@ -130,8 +132,8 @@ fi
echo echo
echo "Configuring ${runner_name} @ $runner_url" echo "Configuring ${runner_name} @ $runner_url"
echo "./config.sh --unattended --url $runner_url --token *** --name $runner_name" echo "./config.sh --unattended --url $runner_url --token *** --name $runner_name --labels $labels"
sudo -E -u ${svc_user} ./config.sh --unattended --url $runner_url --token $RUNNER_TOKEN --name $runner_name sudo -E -u ${svc_user} ./config.sh --unattended --url $runner_url --token $RUNNER_TOKEN --name $runner_name --labels $labels
#--------------------------------------- #---------------------------------------
# Configuring as a service # Configuring as a service

10
src/.editorconfig Normal file
View File

@@ -0,0 +1,10 @@
[*.cs]
charset = utf-8
insert_final_newline = true
csharp_new_line_before_else = true
csharp_new_line_before_catch = true
csharp_new_line_before_finally = true
csharp_new_line_before_open_brace = all
csharp_space_after_keywords_in_control_flow_statements = true

View File

@@ -1,4 +1,4 @@

Microsoft Visual Studio Solution File, Format Version 12.00 Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16 # Visual Studio Version 16
VisualStudioVersion = 16.0.29411.138 VisualStudioVersion = 16.0.29411.138
@@ -21,6 +21,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Sdk", "Sdk\Sdk.csproj", "{D
EndProject EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Test", "Test\Test.csproj", "{C932061F-F6A1-4F1E-B854-A6C6B30DC3EF}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Test", "Test\Test.csproj", "{C932061F-F6A1-4F1E-B854-A6C6B30DC3EF}"
EndProject EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{EFB254FC-7927-445E-BA64-6676ADB309E9}"
ProjectSection(SolutionItems) = preProject
.editorconfig = .editorconfig
EndProjectSection
EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU Debug|Any CPU = Debug|Any CPU

View File

@@ -23,8 +23,6 @@
Default: latest Default: latest
Represents a build version on specific channel. Possible values: Represents a build version on specific channel. Possible values:
- latest - most latest build on specific channel - latest - most latest build on specific channel
- coherent - most latest coherent build on specific channel
coherent applies only to SDK downloads
- 3-part version in a format A.B.C - represents specific version of build - 3-part version in a format A.B.C - represents specific version of build
examples: 2.0.0-preview2-006120, 1.1.0 examples: 2.0.0-preview2-006120, 1.1.0
.PARAMETER InstallDir .PARAMETER InstallDir
@@ -69,6 +67,8 @@
.PARAMETER ProxyUseDefaultCredentials .PARAMETER ProxyUseDefaultCredentials
Default: false Default: false
Use default credentials, when using proxy address. Use default credentials, when using proxy address.
.PARAMETER ProxyBypassList
If set with ProxyAddress, will provide the list of comma separated urls that will bypass the proxy
.PARAMETER SkipNonVersionedFiles .PARAMETER SkipNonVersionedFiles
Default: false Default: false
Skips installing non-versioned files if they already exist, such as dotnet.exe. Skips installing non-versioned files if they already exist, such as dotnet.exe.
@@ -96,6 +96,7 @@ param(
[string]$FeedCredential, [string]$FeedCredential,
[string]$ProxyAddress, [string]$ProxyAddress,
[switch]$ProxyUseDefaultCredentials, [switch]$ProxyUseDefaultCredentials,
[string[]]$ProxyBypassList=@(),
[switch]$SkipNonVersionedFiles, [switch]$SkipNonVersionedFiles,
[switch]$NoCdn [switch]$NoCdn
) )
@@ -119,11 +120,45 @@ $VersionRegEx="/\d+\.\d+[^/]+/"
$OverrideNonVersionedFiles = !$SkipNonVersionedFiles $OverrideNonVersionedFiles = !$SkipNonVersionedFiles
function Say($str) { function Say($str) {
Write-Host "dotnet-install: $str" try {
Write-Host "dotnet-install: $str"
}
catch {
# Some platforms cannot utilize Write-Host (Azure Functions, for instance). Fall back to Write-Output
Write-Output "dotnet-install: $str"
}
}
function Say-Warning($str) {
try {
Write-Warning "dotnet-install: $str"
}
catch {
# Some platforms cannot utilize Write-Warning (Azure Functions, for instance). Fall back to Write-Output
Write-Output "dotnet-install: Warning: $str"
}
}
# Writes a line with error style settings.
# Use this function to show a human-readable comment along with an exception.
function Say-Error($str) {
try {
# Write-Error is quite oververbose for the purpose of the function, let's write one line with error style settings.
$Host.UI.WriteErrorLine("dotnet-install: $str")
}
catch {
Write-Output "dotnet-install: Error: $str"
}
} }
function Say-Verbose($str) { function Say-Verbose($str) {
Write-Verbose "dotnet-install: $str" try {
Write-Verbose "dotnet-install: $str"
}
catch {
# Some platforms cannot utilize Write-Verbose (Azure Functions, for instance). Fall back to Write-Output
Write-Output "dotnet-install: $str"
}
} }
function Say-Invocation($Invocation) { function Say-Invocation($Invocation) {
@@ -137,7 +172,7 @@ function Invoke-With-Retry([ScriptBlock]$ScriptBlock, [int]$MaxAttempts = 3, [in
while ($true) { while ($true) {
try { try {
return $ScriptBlock.Invoke() return & $ScriptBlock
} }
catch { catch {
$Attempts++ $Attempts++
@@ -154,7 +189,16 @@ function Invoke-With-Retry([ScriptBlock]$ScriptBlock, [int]$MaxAttempts = 3, [in
function Get-Machine-Architecture() { function Get-Machine-Architecture() {
Say-Invocation $MyInvocation Say-Invocation $MyInvocation
# possible values: amd64, x64, x86, arm64, arm # On PS x86, PROCESSOR_ARCHITECTURE reports x86 even on x64 systems.
# To get the correct architecture, we need to use PROCESSOR_ARCHITEW6432.
# PS x64 doesn't define this, so we fall back to PROCESSOR_ARCHITECTURE.
# Possible values: amd64, x64, x86, arm64, arm
if( $ENV:PROCESSOR_ARCHITEW6432 -ne $null )
{
return $ENV:PROCESSOR_ARCHITEW6432
}
return $ENV:PROCESSOR_ARCHITECTURE return $ENV:PROCESSOR_ARCHITECTURE
} }
@@ -167,7 +211,7 @@ function Get-CLIArchitecture-From-Architecture([string]$Architecture) {
{ $_ -eq "x86" } { return "x86" } { $_ -eq "x86" } { return "x86" }
{ $_ -eq "arm" } { return "arm" } { $_ -eq "arm" } { return "arm" }
{ $_ -eq "arm64" } { return "arm64" } { $_ -eq "arm64" } { return "arm64" }
default { throw "Architecture not supported. If you think this is a bug, report it at https://github.com/dotnet/sdk/issues" } default { throw "Architecture '$Architecture' not supported. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues" }
} }
} }
@@ -228,7 +272,11 @@ function GetHTTPResponse([Uri] $Uri)
if($ProxyAddress) { if($ProxyAddress) {
$HttpClientHandler = New-Object System.Net.Http.HttpClientHandler $HttpClientHandler = New-Object System.Net.Http.HttpClientHandler
$HttpClientHandler.Proxy = New-Object System.Net.WebProxy -Property @{Address=$ProxyAddress;UseDefaultCredentials=$ProxyUseDefaultCredentials} $HttpClientHandler.Proxy = New-Object System.Net.WebProxy -Property @{
Address=$ProxyAddress;
UseDefaultCredentials=$ProxyUseDefaultCredentials;
BypassList = $ProxyBypassList;
}
$HttpClient = New-Object System.Net.Http.HttpClient -ArgumentList $HttpClientHandler $HttpClient = New-Object System.Net.Http.HttpClient -ArgumentList $HttpClientHandler
} }
else { else {
@@ -238,18 +286,41 @@ function GetHTTPResponse([Uri] $Uri)
# Default timeout for HttpClient is 100s. For a 50 MB download this assumes 500 KB/s average, any less will time out # Default timeout for HttpClient is 100s. For a 50 MB download this assumes 500 KB/s average, any less will time out
# 20 minutes allows it to work over much slower connections. # 20 minutes allows it to work over much slower connections.
$HttpClient.Timeout = New-TimeSpan -Minutes 20 $HttpClient.Timeout = New-TimeSpan -Minutes 20
$Response = $HttpClient.GetAsync("${Uri}${FeedCredential}").Result $Task = $HttpClient.GetAsync("${Uri}${FeedCredential}").ConfigureAwait("false");
if (($Response -eq $null) -or (-not ($Response.IsSuccessStatusCode))) { $Response = $Task.GetAwaiter().GetResult();
# The feed credential is potentially sensitive info. Do not log FeedCredential to console output.
$ErrorMsg = "Failed to download $Uri." if (($null -eq $Response) -or (-not ($Response.IsSuccessStatusCode))) {
if ($Response -ne $null) { # The feed credential is potentially sensitive info. Do not log FeedCredential to console output.
$ErrorMsg += " $Response" $DownloadException = [System.Exception] "Unable to download $Uri."
if ($null -ne $Response) {
$DownloadException.Data["StatusCode"] = [int] $Response.StatusCode
$DownloadException.Data["ErrorMessage"] = "Unable to download $Uri. Returned HTTP status code: " + $DownloadException.Data["StatusCode"]
} }
throw $ErrorMsg throw $DownloadException
} }
return $Response return $Response
}
catch [System.Net.Http.HttpRequestException] {
$DownloadException = [System.Exception] "Unable to download $Uri."
# Pick up the exception message and inner exceptions' messages if they exist
$CurrentException = $PSItem.Exception
$ErrorMsg = $CurrentException.Message + "`r`n"
while ($CurrentException.InnerException) {
$CurrentException = $CurrentException.InnerException
$ErrorMsg += $CurrentException.Message + "`r`n"
}
# Check if there is an issue concerning TLS.
if ($ErrorMsg -like "*SSL/TLS*") {
$ErrorMsg += "Ensure that TLS 1.2 or higher is enabled to use this script.`r`n"
}
$DownloadException.Data["ErrorMessage"] = $ErrorMsg
throw $DownloadException
} }
finally { finally {
if ($HttpClient -ne $null) { if ($HttpClient -ne $null) {
@@ -259,7 +330,7 @@ function GetHTTPResponse([Uri] $Uri)
}) })
} }
function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel, [bool]$Coherent) { function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel) {
Say-Invocation $MyInvocation Say-Invocation $MyInvocation
$VersionFileUrl = $null $VersionFileUrl = $null
@@ -274,12 +345,7 @@ function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel, [bool]$Co
$VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version" $VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version"
} }
elseif (-not $Runtime) { elseif (-not $Runtime) {
if ($Coherent) { $VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.version"
$VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.coherent.version"
}
else {
$VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.version"
}
} }
else { else {
throw "Invalid value for `$Runtime" throw "Invalid value for `$Runtime"
@@ -288,7 +354,8 @@ function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel, [bool]$Co
$Response = GetHTTPResponse -Uri $VersionFileUrl $Response = GetHTTPResponse -Uri $VersionFileUrl
} }
catch { catch {
throw "Could not resolve version information." Say-Error "Could not resolve version information."
throw
} }
$StringContent = $Response.Content.ReadAsStringAsync().Result $StringContent = $Response.Content.ReadAsStringAsync().Result
@@ -314,7 +381,8 @@ function Parse-Jsonfile-For-Version([string]$JSonFile) {
$JSonContent = Get-Content($JSonFile) -Raw | ConvertFrom-Json | Select-Object -expand "sdk" -ErrorAction SilentlyContinue $JSonContent = Get-Content($JSonFile) -Raw | ConvertFrom-Json | Select-Object -expand "sdk" -ErrorAction SilentlyContinue
} }
catch { catch {
throw "Json file unreadable: '$JSonFile'" Say-Error "Json file unreadable: '$JSonFile'"
throw
} }
if ($JSonContent) { if ($JSonContent) {
try { try {
@@ -327,7 +395,8 @@ function Parse-Jsonfile-For-Version([string]$JSonFile) {
} }
} }
catch { catch {
throw "Unable to parse the SDK node in '$JSonFile'" Say-Error "Unable to parse the SDK node in '$JSonFile'"
throw
} }
} }
else { else {
@@ -343,16 +412,12 @@ function Get-Specific-Version-From-Version([string]$AzureFeed, [string]$Channel,
Say-Invocation $MyInvocation Say-Invocation $MyInvocation
if (-not $JSonFile) { if (-not $JSonFile) {
switch ($Version.ToLower()) { if ($Version.ToLower() -eq "latest") {
{ $_ -eq "latest" } { $LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel
$LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $False return $LatestVersionInfo.Version
return $LatestVersionInfo.Version }
} else {
{ $_ -eq "coherent" } { return $Version
$LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $True
return $LatestVersionInfo.Version
}
default { return $Version }
} }
} }
else { else {
@@ -363,17 +428,20 @@ function Get-Specific-Version-From-Version([string]$AzureFeed, [string]$Channel,
function Get-Download-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) { function Get-Download-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) {
Say-Invocation $MyInvocation Say-Invocation $MyInvocation
# If anything fails in this lookup it will default to $SpecificVersion
$SpecificProductVersion = Get-Product-Version -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion
if ($Runtime -eq "dotnet") { if ($Runtime -eq "dotnet") {
$PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-runtime-$SpecificVersion-win-$CLIArchitecture.zip" $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip"
} }
elseif ($Runtime -eq "aspnetcore") { elseif ($Runtime -eq "aspnetcore") {
$PayloadURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/aspnetcore-runtime-$SpecificVersion-win-$CLIArchitecture.zip" $PayloadURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/aspnetcore-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip"
} }
elseif ($Runtime -eq "windowsdesktop") { elseif ($Runtime -eq "windowsdesktop") {
$PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/windowsdesktop-runtime-$SpecificVersion-win-$CLIArchitecture.zip" $PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/windowsdesktop-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip"
} }
elseif (-not $Runtime) { elseif (-not $Runtime) {
$PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-sdk-$SpecificVersion-win-$CLIArchitecture.zip" $PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-sdk-$SpecificProductVersion-win-$CLIArchitecture.zip"
} }
else { else {
throw "Invalid value for `$Runtime" throw "Invalid value for `$Runtime"
@@ -381,7 +449,7 @@ function Get-Download-Link([string]$AzureFeed, [string]$SpecificVersion, [string
Say-Verbose "Constructed primary named payload URL: $PayloadURL" Say-Verbose "Constructed primary named payload URL: $PayloadURL"
return $PayloadURL return $PayloadURL, $SpecificProductVersion
} }
function Get-LegacyDownload-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) { function Get-LegacyDownload-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) {
@@ -402,6 +470,51 @@ function Get-LegacyDownload-Link([string]$AzureFeed, [string]$SpecificVersion, [
return $PayloadURL return $PayloadURL
} }
function Get-Product-Version([string]$AzureFeed, [string]$SpecificVersion) {
Say-Invocation $MyInvocation
if ($Runtime -eq "dotnet") {
$ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/productVersion.txt"
}
elseif ($Runtime -eq "aspnetcore") {
$ProductVersionTxtURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/productVersion.txt"
}
elseif ($Runtime -eq "windowsdesktop") {
$ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/productVersion.txt"
}
elseif (-not $Runtime) {
$ProductVersionTxtURL = "$AzureFeed/Sdk/$SpecificVersion/productVersion.txt"
}
else {
throw "Invalid value '$Runtime' specified for `$Runtime"
}
Say-Verbose "Checking for existence of $ProductVersionTxtURL"
try {
$productVersionResponse = GetHTTPResponse($productVersionTxtUrl)
if ($productVersionResponse.StatusCode -eq 200) {
$productVersion = $productVersionResponse.Content.ReadAsStringAsync().Result.Trim()
if ($productVersion -ne $SpecificVersion)
{
Say "Using alternate version $productVersion found in $ProductVersionTxtURL"
}
return $productVersion
}
else {
Say-Verbose "Got StatusCode $($productVersionResponse.StatusCode) trying to get productVersion.txt at $productVersionTxtUrl, so using default value of $SpecificVersion"
$productVersion = $SpecificVersion
}
} catch {
Say-Verbose "Could not read productVersion.txt at $productVersionTxtUrl, so using default value of $SpecificVersion (Exception: '$($_.Exception.Message)' )"
$productVersion = $SpecificVersion
}
return $productVersion
}
function Get-User-Share-Path() { function Get-User-Share-Path() {
Say-Invocation $MyInvocation Say-Invocation $MyInvocation
@@ -539,6 +652,23 @@ function DownloadFile($Source, [string]$OutPath) {
} }
} }
function SafeRemoveFile($Path) {
try {
if (Test-Path $Path) {
Remove-Item $Path
Say-Verbose "The temporary file `"$Path`" was removed."
}
else
{
Say-Verbose "The temporary file `"$Path`" does not exist, therefore is not removed."
}
}
catch
{
Say-Warning "Failed to remove the temporary file: `"$Path`", remove it manually."
}
}
function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot, [string]$BinFolderRelativePath) { function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot, [string]$BinFolderRelativePath) {
$BinPath = Get-Absolute-Path $(Join-Path -Path $InstallRoot -ChildPath $BinFolderRelativePath) $BinPath = Get-Absolute-Path $(Join-Path -Path $InstallRoot -ChildPath $BinFolderRelativePath)
if (-Not $NoPath) { if (-Not $NoPath) {
@@ -555,9 +685,14 @@ function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot, [string]$BinFolde
} }
} }
Say "Note that the intended use of this script is for Continuous Integration (CI) scenarios, where:"
Say "- The SDK needs to be installed without user interaction and without admin rights."
Say "- The SDK installation doesn't need to persist across multiple CI runs."
Say "To set up a development environment or to run apps, use installers rather than this script. Visit https://dotnet.microsoft.com/download to get the installer.`r`n"
$CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture $CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture
$SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $AzureFeed -Channel $Channel -Version $Version -JSonFile $JSonFile $SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $AzureFeed -Channel $Channel -Version $Version -JSonFile $JSonFile
$DownloadLink = Get-Download-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture $DownloadLink, $EffectiveVersion = Get-Download-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture
$LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture $LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture
$InstallRoot = Resolve-Installation-Path $InstallDir $InstallRoot = Resolve-Installation-Path $InstallDir
@@ -583,7 +718,12 @@ if ($DryRun) {
} }
} }
Say "Repeatable invocation: $RepeatableCommand" Say "Repeatable invocation: $RepeatableCommand"
exit 0 if ($SpecificVersion -ne $EffectiveVersion)
{
Say "NOTE: Due to finding a version manifest with this runtime, it would actually install with version '$EffectiveVersion'"
}
return
} }
if ($Runtime -eq "dotnet") { if ($Runtime -eq "dotnet") {
@@ -606,12 +746,18 @@ else {
throw "Invalid value for `$Runtime" throw "Invalid value for `$Runtime"
} }
if ($SpecificVersion -ne $EffectiveVersion)
{
Say "Performing installation checks for effective version: $EffectiveVersion"
$SpecificVersion = $EffectiveVersion
}
# Check if the SDK version is already installed. # Check if the SDK version is already installed.
$isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion
if ($isAssetInstalled) { if ($isAssetInstalled) {
Say "$assetName version $SpecificVersion is already installed." Say "$assetName version $SpecificVersion is already installed."
Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath
exit 0 return
} }
New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null
@@ -619,30 +765,69 @@ New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null
$installDrive = $((Get-Item $InstallRoot).PSDrive.Name); $installDrive = $((Get-Item $InstallRoot).PSDrive.Name);
$diskInfo = Get-PSDrive -Name $installDrive $diskInfo = Get-PSDrive -Name $installDrive
if ($diskInfo.Free / 1MB -le 100) { if ($diskInfo.Free / 1MB -le 100) {
Say "There is not enough disk space on drive ${installDrive}:" throw "There is not enough disk space on drive ${installDrive}:"
exit 0
} }
$ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName())
Say-Verbose "Zip path: $ZipPath" Say-Verbose "Zip path: $ZipPath"
$DownloadFailed = $false $DownloadFailed = $false
Say "Downloading link: $DownloadLink"
$PrimaryDownloadStatusCode = 0
$LegacyDownloadStatusCode = 0
$PrimaryDownloadFailedMsg = ""
$LegacyDownloadFailedMsg = ""
Say "Downloading primary link $DownloadLink"
try { try {
DownloadFile -Source $DownloadLink -OutPath $ZipPath DownloadFile -Source $DownloadLink -OutPath $ZipPath
} }
catch { catch {
Say "Cannot download: $DownloadLink" if ($PSItem.Exception.Data.Contains("StatusCode")) {
$PrimaryDownloadStatusCode = $PSItem.Exception.Data["StatusCode"]
}
if ($PSItem.Exception.Data.Contains("ErrorMessage")) {
$PrimaryDownloadFailedMsg = $PSItem.Exception.Data["ErrorMessage"]
} else {
$PrimaryDownloadFailedMsg = $PSItem.Exception.Message
}
if ($PrimaryDownloadStatusCode -eq 404) {
Say "The resource at $DownloadLink is not available."
} else {
Say $PSItem.Exception.Message
}
SafeRemoveFile -Path $ZipPath
if ($LegacyDownloadLink) { if ($LegacyDownloadLink) {
$DownloadLink = $LegacyDownloadLink $DownloadLink = $LegacyDownloadLink
$ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName()) $ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName())
Say-Verbose "Legacy zip path: $ZipPath" Say-Verbose "Legacy zip path: $ZipPath"
Say "Downloading legacy link: $DownloadLink" Say "Downloading legacy link $DownloadLink"
try { try {
DownloadFile -Source $DownloadLink -OutPath $ZipPath DownloadFile -Source $DownloadLink -OutPath $ZipPath
} }
catch { catch {
Say "Cannot download: $DownloadLink" if ($PSItem.Exception.Data.Contains("StatusCode")) {
$LegacyDownloadStatusCode = $PSItem.Exception.Data["StatusCode"]
}
if ($PSItem.Exception.Data.Contains("ErrorMessage")) {
$LegacyDownloadFailedMsg = $PSItem.Exception.Data["ErrorMessage"]
} else {
$LegacyDownloadFailedMsg = $PSItem.Exception.Message
}
if ($LegacyDownloadStatusCode -eq 404) {
Say "The resource at $DownloadLink is not available."
} else {
Say $PSItem.Exception.Message
}
SafeRemoveFile -Path $ZipPath
$DownloadFailed = $true $DownloadFailed = $true
} }
} }
@@ -652,7 +837,19 @@ catch {
} }
if ($DownloadFailed) { if ($DownloadFailed) {
throw "Could not find/download: `"$assetName`" with version = $SpecificVersion`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" if (($PrimaryDownloadStatusCode -eq 404) -and ((-not $LegacyDownloadLink) -or ($LegacyDownloadStatusCode -eq 404))) {
throw "Could not find `"$assetName`" with version = $SpecificVersion`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support"
} else {
# 404-NotFound is an expected response if it goes from only one of the links, do not show that error.
# If primary path is available (not 404-NotFound) then show the primary error else show the legacy error.
if ($PrimaryDownloadStatusCode -ne 404) {
throw "Could not download `"$assetName`" with version = $SpecificVersion`r`n$PrimaryDownloadFailedMsg"
}
if (($LegacyDownloadLink) -and ($LegacyDownloadStatusCode -ne 404)) {
throw "Could not download `"$assetName`" with version = $SpecificVersion`r`n$LegacyDownloadFailedMsg"
}
throw "Could not download `"$assetName`" with version = $SpecificVersion"
}
} }
Say "Extracting zip from $DownloadLink" Say "Extracting zip from $DownloadLink"
@@ -674,13 +871,208 @@ if (!$isAssetInstalled) {
$isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion $isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion
} }
# Version verification failed. More likely something is wrong either with the downloaded content or with the verification algorithm.
if (!$isAssetInstalled) { if (!$isAssetInstalled) {
Say-Error "Failed to verify the version of installed `"$assetName`".`nInstallation source: $DownloadLink.`nInstallation location: $InstallRoot.`nReport the bug at https://github.com/dotnet/install-scripts/issues."
throw "`"$assetName`" with version = $SpecificVersion failed to install with an unknown error." throw "`"$assetName`" with version = $SpecificVersion failed to install with an unknown error."
} }
Remove-Item $ZipPath SafeRemoveFile -Path $ZipPath
Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath
Say "Note that the script does not resolve dependencies during installation."
Say "To check the list of dependencies, go to https://docs.microsoft.com/dotnet/core/install/windows#dependencies"
Say "Installation finished" Say "Installation finished"
exit 0 # SIG # Begin signature block
# MIIjkgYJKoZIhvcNAQcCoIIjgzCCI38CAQExDzANBglghkgBZQMEAgEFADB5Bgor
# BgEEAYI3AgEEoGswaTA0BgorBgEEAYI3AgEeMCYCAwEAAAQQH8w7YFlLCE63JNLG
# KX7zUQIBAAIBAAIBAAIBAAIBADAxMA0GCWCGSAFlAwQCAQUABCD2c707qnCLOLIC
# n6Mu5Gr4+Xp68foyZlGlTycnycc5l6CCDYEwggX/MIID56ADAgECAhMzAAABh3IX
# chVZQMcJAAAAAAGHMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNVBAYTAlVTMRMwEQYD
# VQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMTH01pY3Jvc29mdCBDb2RlIFNpZ25p
# bmcgUENBIDIwMTEwHhcNMjAwMzA0MTgzOTQ3WhcNMjEwMzAzMTgzOTQ3WjB0MQsw
# CQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9u
# ZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMR4wHAYDVQQDExVNaWNy
# b3NvZnQgQ29ycG9yYXRpb24wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
# AQDOt8kLc7P3T7MKIhouYHewMFmnq8Ayu7FOhZCQabVwBp2VS4WyB2Qe4TQBT8aB
# znANDEPjHKNdPT8Xz5cNali6XHefS8i/WXtF0vSsP8NEv6mBHuA2p1fw2wB/F0dH
# sJ3GfZ5c0sPJjklsiYqPw59xJ54kM91IOgiO2OUzjNAljPibjCWfH7UzQ1TPHc4d
# weils8GEIrbBRb7IWwiObL12jWT4Yh71NQgvJ9Fn6+UhD9x2uk3dLj84vwt1NuFQ
# itKJxIV0fVsRNR3abQVOLqpDugbr0SzNL6o8xzOHL5OXiGGwg6ekiXA1/2XXY7yV
# Fc39tledDtZjSjNbex1zzwSXAgMBAAGjggF+MIIBejAfBgNVHSUEGDAWBgorBgEE
# AYI3TAgBBggrBgEFBQcDAzAdBgNVHQ4EFgQUhov4ZyO96axkJdMjpzu2zVXOJcsw
# UAYDVR0RBEkwR6RFMEMxKTAnBgNVBAsTIE1pY3Jvc29mdCBPcGVyYXRpb25zIFB1
# ZXJ0byBSaWNvMRYwFAYDVQQFEw0yMzAwMTIrNDU4Mzg1MB8GA1UdIwQYMBaAFEhu
# ZOVQBdOCqhc3NyK1bajKdQKVMFQGA1UdHwRNMEswSaBHoEWGQ2h0dHA6Ly93d3cu
# bWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY0NvZFNpZ1BDQTIwMTFfMjAxMS0w
# Ny0wOC5jcmwwYQYIKwYBBQUHAQEEVTBTMFEGCCsGAQUFBzAChkVodHRwOi8vd3d3
# Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2NlcnRzL01pY0NvZFNpZ1BDQTIwMTFfMjAx
# MS0wNy0wOC5jcnQwDAYDVR0TAQH/BAIwADANBgkqhkiG9w0BAQsFAAOCAgEAixmy
# S6E6vprWD9KFNIB9G5zyMuIjZAOuUJ1EK/Vlg6Fb3ZHXjjUwATKIcXbFuFC6Wr4K
# NrU4DY/sBVqmab5AC/je3bpUpjtxpEyqUqtPc30wEg/rO9vmKmqKoLPT37svc2NV
# BmGNl+85qO4fV/w7Cx7J0Bbqk19KcRNdjt6eKoTnTPHBHlVHQIHZpMxacbFOAkJr
# qAVkYZdz7ikNXTxV+GRb36tC4ByMNxE2DF7vFdvaiZP0CVZ5ByJ2gAhXMdK9+usx
# zVk913qKde1OAuWdv+rndqkAIm8fUlRnr4saSCg7cIbUwCCf116wUJ7EuJDg0vHe
# yhnCeHnBbyH3RZkHEi2ofmfgnFISJZDdMAeVZGVOh20Jp50XBzqokpPzeZ6zc1/g
# yILNyiVgE+RPkjnUQshd1f1PMgn3tns2Cz7bJiVUaqEO3n9qRFgy5JuLae6UweGf
# AeOo3dgLZxikKzYs3hDMaEtJq8IP71cX7QXe6lnMmXU/Hdfz2p897Zd+kU+vZvKI
# 3cwLfuVQgK2RZ2z+Kc3K3dRPz2rXycK5XCuRZmvGab/WbrZiC7wJQapgBodltMI5
# GMdFrBg9IeF7/rP4EqVQXeKtevTlZXjpuNhhjuR+2DMt/dWufjXpiW91bo3aH6Ea
# jOALXmoxgltCp1K7hrS6gmsvj94cLRf50QQ4U8Qwggd6MIIFYqADAgECAgphDpDS
# AAAAAAADMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMK
# V2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0
# IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0
# ZSBBdXRob3JpdHkgMjAxMTAeFw0xMTA3MDgyMDU5MDlaFw0yNjA3MDgyMTA5MDla
# MH4xCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdS
# ZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKDAmBgNVBAMT
# H01pY3Jvc29mdCBDb2RlIFNpZ25pbmcgUENBIDIwMTEwggIiMA0GCSqGSIb3DQEB
# AQUAA4ICDwAwggIKAoICAQCr8PpyEBwurdhuqoIQTTS68rZYIZ9CGypr6VpQqrgG
# OBoESbp/wwwe3TdrxhLYC/A4wpkGsMg51QEUMULTiQ15ZId+lGAkbK+eSZzpaF7S
# 35tTsgosw6/ZqSuuegmv15ZZymAaBelmdugyUiYSL+erCFDPs0S3XdjELgN1q2jz
# y23zOlyhFvRGuuA4ZKxuZDV4pqBjDy3TQJP4494HDdVceaVJKecNvqATd76UPe/7
# 4ytaEB9NViiienLgEjq3SV7Y7e1DkYPZe7J7hhvZPrGMXeiJT4Qa8qEvWeSQOy2u
# M1jFtz7+MtOzAz2xsq+SOH7SnYAs9U5WkSE1JcM5bmR/U7qcD60ZI4TL9LoDho33
# X/DQUr+MlIe8wCF0JV8YKLbMJyg4JZg5SjbPfLGSrhwjp6lm7GEfauEoSZ1fiOIl
# XdMhSz5SxLVXPyQD8NF6Wy/VI+NwXQ9RRnez+ADhvKwCgl/bwBWzvRvUVUvnOaEP
# 6SNJvBi4RHxF5MHDcnrgcuck379GmcXvwhxX24ON7E1JMKerjt/sW5+v/N2wZuLB
# l4F77dbtS+dJKacTKKanfWeA5opieF+yL4TXV5xcv3coKPHtbcMojyyPQDdPweGF
# RInECUzF1KVDL3SV9274eCBYLBNdYJWaPk8zhNqwiBfenk70lrC8RqBsmNLg1oiM
# CwIDAQABo4IB7TCCAekwEAYJKwYBBAGCNxUBBAMCAQAwHQYDVR0OBBYEFEhuZOVQ
# BdOCqhc3NyK1bajKdQKVMBkGCSsGAQQBgjcUAgQMHgoAUwB1AGIAQwBBMAsGA1Ud
# DwQEAwIBhjAPBgNVHRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFHItOgIxkEO5FAVO
# 4eqnxzHRI4k0MFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6Ly9jcmwubWljcm9zb2Z0
# LmNvbS9wa2kvY3JsL3Byb2R1Y3RzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y
# Mi5jcmwwXgYIKwYBBQUHAQEEUjBQME4GCCsGAQUFBzAChkJodHRwOi8vd3d3Lm1p
# Y3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dDIwMTFfMjAxMV8wM18y
# Mi5jcnQwgZ8GA1UdIASBlzCBlDCBkQYJKwYBBAGCNy4DMIGDMD8GCCsGAQUFBwIB
# FjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL2RvY3MvcHJpbWFyeWNw
# cy5odG0wQAYIKwYBBQUHAgIwNB4yIB0ATABlAGcAYQBsAF8AcABvAGwAaQBjAHkA
# XwBzAHQAYQB0AGUAbQBlAG4AdAAuIB0wDQYJKoZIhvcNAQELBQADggIBAGfyhqWY
# 4FR5Gi7T2HRnIpsLlhHhY5KZQpZ90nkMkMFlXy4sPvjDctFtg/6+P+gKyju/R6mj
# 82nbY78iNaWXXWWEkH2LRlBV2AySfNIaSxzzPEKLUtCw/WvjPgcuKZvmPRul1LUd
# d5Q54ulkyUQ9eHoj8xN9ppB0g430yyYCRirCihC7pKkFDJvtaPpoLpWgKj8qa1hJ
# Yx8JaW5amJbkg/TAj/NGK978O9C9Ne9uJa7lryft0N3zDq+ZKJeYTQ49C/IIidYf
# wzIY4vDFLc5bnrRJOQrGCsLGra7lstnbFYhRRVg4MnEnGn+x9Cf43iw6IGmYslmJ
# aG5vp7d0w0AFBqYBKig+gj8TTWYLwLNN9eGPfxxvFX1Fp3blQCplo8NdUmKGwx1j
# NpeG39rz+PIWoZon4c2ll9DuXWNB41sHnIc+BncG0QaxdR8UvmFhtfDcxhsEvt9B
# xw4o7t5lL+yX9qFcltgA1qFGvVnzl6UJS0gQmYAf0AApxbGbpT9Fdx41xtKiop96
# eiL6SJUfq/tHI4D1nvi/a7dLl+LrdXga7Oo3mXkYS//WsyNodeav+vyL6wuA6mk7
# r/ww7QRMjt/fdW1jkT3RnVZOT7+AVyKheBEyIXrvQQqxP/uozKRdwaGIm1dxVk5I
# RcBCyZt2WwqASGv9eZ/BvW1taslScxMNelDNMYIVZzCCFWMCAQEwgZUwfjELMAkG
# A1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQx
# HjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEoMCYGA1UEAxMfTWljcm9z
# b2Z0IENvZGUgU2lnbmluZyBQQ0EgMjAxMQITMwAAAYdyF3IVWUDHCQAAAAABhzAN
# BglghkgBZQMEAgEFAKCBrjAZBgkqhkiG9w0BCQMxDAYKKwYBBAGCNwIBBDAcBgor
# BgEEAYI3AgELMQ4wDAYKKwYBBAGCNwIBFTAvBgkqhkiG9w0BCQQxIgQgE/MRhWyu
# Zg+EA2WKcxYC31nHVCTE6guHppZppc70RtkwQgYKKwYBBAGCNwIBDDE0MDKgFIAS
# AE0AaQBjAHIAbwBzAG8AZgB0oRqAGGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbTAN
# BgkqhkiG9w0BAQEFAASCAQBvcYCjRDXUYEIz9j2j0r4GFI2Y3g/CoNxDDBaeQ+gV
# khO0fK0oLh18RbV271Mg6SF7X7+mXB5MnL68voVQDqHnsCYrIAuMF/AEpv9YuDDp
# ZRJuqN7Vwg3HM02l/FyATBIMgf/V79aYzJL3jjtt9bRIyxk6aPU4XcwMeA4usnUQ
# rMhIiQz07DgfSrcQWe4AvGFAIvqTAKE4P944EZWWVnWI/10rvatEAefqJZX3XljW
# sK/6NY/0MyAyiILOuXbvVS0YFbHaR2qd1jUXbrY79fS+H4Ts6qnbufOkHQvmcDxs
# 801wKLHumMdPTtMVzfVMCwPvrHP0wtzsFlmCcKjBbGpvoYIS8TCCEu0GCisGAQQB
# gjcDAwExghLdMIIS2QYJKoZIhvcNAQcCoIISyjCCEsYCAQMxDzANBglghkgBZQME
# AgEFADCCAVUGCyqGSIb3DQEJEAEEoIIBRASCAUAwggE8AgEBBgorBgEEAYRZCgMB
# MDEwDQYJYIZIAWUDBAIBBQAEINdeoXtuzW+Dihw6n+VdG+91si0f6TvWhJXaPtvW
# oF4cAgZfu+i3IT8YEzIwMjAxMjE3MDYzMDM2LjU0M1owBIACAfSggdSkgdEwgc4x
# CzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRt
# b25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKTAnBgNVBAsTIE1p
# Y3Jvc29mdCBPcGVyYXRpb25zIFB1ZXJ0byBSaWNvMSYwJAYDVQQLEx1UaGFsZXMg
# VFNTIEVTTjo4OTdBLUUzNTYtMTcwMTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUt
# U3RhbXAgU2VydmljZaCCDkQwggT1MIID3aADAgECAhMzAAABLCKvRZd1+RvuAAAA
# AAEsMA0GCSqGSIb3DQEBCwUAMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNo
# aW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29y
# cG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEw
# MB4XDTE5MTIxOTAxMTUwM1oXDTIxMDMxNzAxMTUwM1owgc4xCzAJBgNVBAYTAlVT
# MRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQK
# ExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKTAnBgNVBAsTIE1pY3Jvc29mdCBPcGVy
# YXRpb25zIFB1ZXJ0byBSaWNvMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjo4OTdB
# LUUzNTYtMTcwMTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vydmlj
# ZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAPK1zgSSq+MxAYo3qpCt
# QDxSMPPJy6mm/wfEJNjNUnYtLFBwl1BUS5trEk/t41ldxITKehs+ABxYqo4Qxsg3
# Gy1ugKiwHAnYiiekfC+ZhptNFgtnDZIn45zC0AlVr/6UfLtsLcHCh1XElLUHfEC0
# nBuQcM/SpYo9e3l1qY5NdMgDGxCsmCKdiZfYXIu+U0UYIBhdzmSHnB3fxZOBVcr5
# htFHEBBNt/rFJlm/A4yb8oBsp+Uf0p5QwmO/bCcdqB15JpylOhZmWs0sUfJKlK9E
# rAhBwGki2eIRFKsQBdkXS9PWpF1w2gIJRvSkDEaCf+lbGTPdSzHSbfREWOF9wY3i
# Yj8CAwEAAaOCARswggEXMB0GA1UdDgQWBBRRahZSGfrCQhCyIyGH9DkiaW7L0zAf
# BgNVHSMEGDAWgBTVYzpcijGQ80N7fEYbxTNoWoVtVTBWBgNVHR8ETzBNMEugSaBH
# hkVodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNU
# aW1TdGFQQ0FfMjAxMC0wNy0wMS5jcmwwWgYIKwYBBQUHAQEETjBMMEoGCCsGAQUF
# BzAChj5odHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpL2NlcnRzL01pY1RpbVN0
# YVBDQV8yMDEwLTA3LTAxLmNydDAMBgNVHRMBAf8EAjAAMBMGA1UdJQQMMAoGCCsG
# AQUFBwMIMA0GCSqGSIb3DQEBCwUAA4IBAQBPFxHIwi4vAH49w9Svmz6K3tM55RlW
# 5pPeULXdut2Rqy6Ys0+VpZsbuaEoxs6Z1C3hMbkiqZFxxyltxJpuHTyGTg61zfNI
# F5n6RsYF3s7IElDXNfZznF1/2iWc6uRPZK8rxxUJ/7emYXZCYwuUY0XjsCpP9pbR
# RKeJi6r5arSyI+NfKxvgoM21JNt1BcdlXuAecdd/k8UjxCscffanoK2n6LFw1PcZ
# lEO7NId7o+soM2C0QY5BYdghpn7uqopB6ixyFIIkDXFub+1E7GmAEwfU6VwEHL7y
# 9rNE8bd+JrQs+yAtkkHy9FmXg/PsGq1daVzX1So7CJ6nyphpuHSN3VfTMIIGcTCC
# BFmgAwIBAgIKYQmBKgAAAAAAAjANBgkqhkiG9w0BAQsFADCBiDELMAkGA1UEBhMC
# VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV
# BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEyMDAGA1UEAxMpTWljcm9zb2Z0IFJv
# b3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTAwHhcNMTAwNzAxMjEzNjU1WhcN
# MjUwNzAxMjE0NjU1WjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3Rv
# bjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0
# aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDCCASIw
# DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKkdDbx3EYo6IOz8E5f1+n9plGt0
# VBDVpQoAgoX77XxoSyxfxcPlYcJ2tz5mK1vwFVMnBDEfQRsalR3OCROOfGEwWbEw
# RA/xYIiEVEMM1024OAizQt2TrNZzMFcmgqNFDdDq9UeBzb8kYDJYYEbyWEeGMoQe
# dGFnkV+BVLHPk0ySwcSmXdFhE24oxhr5hoC732H8RsEnHSRnEnIaIYqvS2SJUGKx
# Xf13Hz3wV3WsvYpCTUBR0Q+cBj5nf/VmwAOWRH7v0Ev9buWayrGo8noqCjHw2k4G
# kbaICDXoeByw6ZnNPOcvRLqn9NxkvaQBwSAJk3jN/LzAyURdXhacAQVPIk0CAwEA
# AaOCAeYwggHiMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBTVYzpcijGQ80N7
# fEYbxTNoWoVtVTAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMC
# AYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBTV9lbLj+iiXGJo0T2UkFvX
# zpoYxDBWBgNVHR8ETzBNMEugSaBHhkVodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20v
# cGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcmwwWgYI
# KwYBBQUHAQEETjBMMEoGCCsGAQUFBzAChj5odHRwOi8vd3d3Lm1pY3Jvc29mdC5j
# b20vcGtpL2NlcnRzL01pY1Jvb0NlckF1dF8yMDEwLTA2LTIzLmNydDCBoAYDVR0g
# AQH/BIGVMIGSMIGPBgkrBgEEAYI3LgMwgYEwPQYIKwYBBQUHAgEWMWh0dHA6Ly93
# d3cubWljcm9zb2Z0LmNvbS9QS0kvZG9jcy9DUFMvZGVmYXVsdC5odG0wQAYIKwYB
# BQUHAgIwNB4yIB0ATABlAGcAYQBsAF8AUABvAGwAaQBjAHkAXwBTAHQAYQB0AGUA
# bQBlAG4AdAAuIB0wDQYJKoZIhvcNAQELBQADggIBAAfmiFEN4sbgmD+BcQM9naOh
# IW+z66bM9TG+zwXiqf76V20ZMLPCxWbJat/15/B4vceoniXj+bzta1RXCCtRgkQS
# +7lTjMz0YBKKdsxAQEGb3FwX/1z5Xhc1mCRWS3TvQhDIr79/xn/yN31aPxzymXlK
# kVIArzgPF/UveYFl2am1a+THzvbKegBvSzBEJCI8z+0DpZaPWSm8tv0E4XCfMkon
# /VWvL/625Y4zu2JfmttXQOnxzplmkIz/amJ/3cVKC5Em4jnsGUpxY517IW3DnKOi
# PPp/fZZqkHimbdLhnPkd/DjYlPTGpQqWhqS9nhquBEKDuLWAmyI4ILUl5WTs9/S/
# fmNZJQ96LjlXdqJxqgaKD4kWumGnEcua2A5HmoDF0M2n0O99g/DhO3EJ3110mCII
# YdqwUB5vvfHhAN/nMQekkzr3ZUd46PioSKv33nJ+YWtvd6mBy6cJrDm77MbL2IK0
# cs0d9LiFAR6A+xuJKlQ5slvayA1VmXqHczsI5pgt6o3gMy4SKfXAL1QnIffIrE7a
# KLixqduWsqdCosnPGUFN4Ib5KpqjEWYw07t0MkvfY3v1mYovG8chr1m1rtxEPJdQ
# cdeh0sVV42neV8HR3jDA/czmTfsNv11P6Z0eGTgvvM9YBS7vDaBQNdrvCScc1bN+
# NR4Iuto229Nfj950iEkSoYIC0jCCAjsCAQEwgfyhgdSkgdEwgc4xCzAJBgNVBAYT
# AlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYD
# VQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xKTAnBgNVBAsTIE1pY3Jvc29mdCBP
# cGVyYXRpb25zIFB1ZXJ0byBSaWNvMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjo4
# OTdBLUUzNTYtMTcwMTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2Vy
# dmljZaIjCgEBMAcGBSsOAwIaAxUADE5OKSMoNx/mYxYWap1RTOohbJ2ggYMwgYCk
# fjB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMH
# UmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQD
# Ex1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDANBgkqhkiG9w0BAQUFAAIF
# AOOFYaowIhgPMjAyMDEyMTcwODQ4NDJaGA8yMDIwMTIxODA4NDg0MlowdzA9Bgor
# BgEEAYRZCgQBMS8wLTAKAgUA44VhqgIBADAKAgEAAgIoWgIB/zAHAgEAAgISJTAK
# AgUA44azKgIBADA2BgorBgEEAYRZCgQCMSgwJjAMBgorBgEEAYRZCgMCoAowCAIB
# AAIDB6EgoQowCAIBAAIDAYagMA0GCSqGSIb3DQEBBQUAA4GBAB53NDoDDF4vqFWY
# fwUnSvAy3z0CtqSFeA9RzDKGklPRwVkya5DtmVBDTZUbVQ2ST9hvRAVxhktfyVBZ
# ewapGJsvwMhg7nnEqBOumt6TvueIZpbs+p5z//3+iFYGkT3YFQI0Gd2JkvgBxfs5
# +GptO6JKtiyA+zkKijxqXZvMqMxBMYIDDTCCAwkCAQEwgZMwfDELMAkGA1UEBhMC
# VVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNV
# BAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRp
# bWUtU3RhbXAgUENBIDIwMTACEzMAAAEsIq9Fl3X5G+4AAAAAASwwDQYJYIZIAWUD
# BAIBBQCgggFKMBoGCSqGSIb3DQEJAzENBgsqhkiG9w0BCRABBDAvBgkqhkiG9w0B
# CQQxIgQg3wEUtEvxwCp3aAFB2vGXOOqg/AXHyXZh9P9J+0uArDMwgfoGCyqGSIb3
# DQEJEAIvMYHqMIHnMIHkMIG9BCBbn/0uFFh42hTM5XOoKdXevBaiSxmYK9Ilcn9n
# u5ZH4TCBmDCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9u
# MRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRp
# b24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwAhMzAAAB
# LCKvRZd1+RvuAAAAAAEsMCIEINBRtGID6jvA2ptfwIuPyG7qPcLRYb9YrJ8aKfVg
# TulFMA0GCSqGSIb3DQEBCwUABIIBACQQpFGWW6JmH5MTKwhaE/8+gyzI2bT8XJnA
# t8k7PHFvEGA7whgp9eNgW+wWJm1gnsmswjx2l7FW4DLg9lghM8FK77JRCg7CJfse
# dSbnTv81/4VhSXOAO0jMP2dALP7DF59vQmlDh50u8/Wu61ActMOt6cArkoUhBRXO
# LnqOQCOEEku5Xy2ES9g9eUfLUvTvlWo6HiAq+cJnNV08QRBOnGWRxdwy8YJ5vwNW
# Pwx0ZG3rTvMtGzOaW6Ve5O36H2ynoEdzCmpakeDaF2sZ86/LNERKyIXiykV/Uig1
# SZh2VLY/Yni9SCVHbYgvTOCh5ZZE5eOi6BwLf0T4xl5alHUx+AA=
# SIG # End signature block

View File

@@ -40,7 +40,7 @@ if [ -t 1 ] && command -v tput > /dev/null; then
fi fi
say_warning() { say_warning() {
printf "%b\n" "${yellow:-}dotnet_install: Warning: $1${normal:-}" printf "%b\n" "${yellow:-}dotnet_install: Warning: $1${normal:-}" >&3
} }
say_err() { say_err() {
@@ -183,6 +183,9 @@ get_current_os_name() {
elif is_musl_based_distro; then elif is_musl_based_distro; then
echo "linux-musl" echo "linux-musl"
return 0 return 0
elif [ "$linux_platform_name" = "linux-musl" ]; then
echo "linux-musl"
return 0
else else
echo "linux" echo "linux"
return 0 return 0
@@ -241,42 +244,6 @@ check_min_reqs() {
return 0 return 0
} }
check_pre_reqs() {
eval $invocation
if [ "${DOTNET_INSTALL_SKIP_PREREQS:-}" = "1" ]; then
return 0
fi
if [ "$(uname)" = "Linux" ]; then
if is_musl_based_distro; then
if ! command -v scanelf > /dev/null; then
say_warning "scanelf not found, please install pax-utils package."
return 0
fi
LDCONFIG_COMMAND="scanelf --ldpath -BF '%f'"
[ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libintl)" ] && say_warning "Unable to locate libintl. Probable prerequisite missing; install libintl (or gettext)."
else
if [ ! -x "$(command -v ldconfig)" ]; then
say_verbose "ldconfig is not in PATH, trying /sbin/ldconfig."
LDCONFIG_COMMAND="/sbin/ldconfig"
else
LDCONFIG_COMMAND="ldconfig"
fi
local librarypath=${LD_LIBRARY_PATH:-}
LDCONFIG_COMMAND="$LDCONFIG_COMMAND -NXv ${librarypath//:/ }"
fi
[ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep zlib)" ] && say_warning "Unable to locate zlib. Probable prerequisite missing; install zlib."
[ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep ssl)" ] && say_warning "Unable to locate libssl. Probable prerequisite missing; install libssl."
[ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libicu)" ] && say_warning "Unable to locate libicu. Probable prerequisite missing; install libicu."
[ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep lttng)" ] && say_warning "Unable to locate liblttng. Probable prerequisite missing; install libcurl."
[ -z "$($LDCONFIG_COMMAND 2>/dev/null | grep libcurl)" ] && say_warning "Unable to locate libcurl. Probable prerequisite missing; install libcurl."
fi
return 0
}
# args: # args:
# input - $1 # input - $1
to_lowercase() { to_lowercase() {
@@ -332,7 +299,7 @@ get_machine_architecture() {
if command -v uname > /dev/null; then if command -v uname > /dev/null; then
CPUName=$(uname -m) CPUName=$(uname -m)
case $CPUName in case $CPUName in
armv7l) armv*l)
echo "arm" echo "arm"
return 0 return 0
;; ;;
@@ -373,10 +340,34 @@ get_normalized_architecture_from_architecture() {
;; ;;
esac esac
say_err "Architecture \`$architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/sdk/issues" say_err "Architecture \`$architecture\` not supported. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues"
return 1 return 1
} }
# args:
# user_defined_os - $1
get_normalized_os() {
eval $invocation
local osname="$(to_lowercase "$1")"
if [ ! -z "$osname" ]; then
case "$osname" in
osx | freebsd | rhel.6 | linux-musl | linux)
echo "$osname"
return 0
;;
*)
say_err "'$user_defined_os' is not a supported value for --os option, supported values are: osx, linux, linux-musl, freebsd, rhel.6. If you think this is a bug, report it at https://github.com/dotnet/install-scripts/issues."
return 1
;;
esac
else
osname="$(get_current_os_name)" || return 1
fi
echo "$osname"
return 0
}
# The version text returned from the feeds is a 1-line or 2-line string: # The version text returned from the feeds is a 1-line or 2-line string:
# For the SDK and the dotnet runtime (2 lines): # For the SDK and the dotnet runtime (2 lines):
# Line 1: # commit_hash # Line 1: # commit_hash
@@ -418,14 +409,12 @@ is_dotnet_package_installed() {
# azure_feed - $1 # azure_feed - $1
# channel - $2 # channel - $2
# normalized_architecture - $3 # normalized_architecture - $3
# coherent - $4
get_latest_version_info() { get_latest_version_info() {
eval $invocation eval $invocation
local azure_feed="$1" local azure_feed="$1"
local channel="$2" local channel="$2"
local normalized_architecture="$3" local normalized_architecture="$3"
local coherent="$4"
local version_file_url=null local version_file_url=null
if [[ "$runtime" == "dotnet" ]]; then if [[ "$runtime" == "dotnet" ]]; then
@@ -433,11 +422,7 @@ get_latest_version_info() {
elif [[ "$runtime" == "aspnetcore" ]]; then elif [[ "$runtime" == "aspnetcore" ]]; then
version_file_url="$uncached_feed/aspnetcore/Runtime/$channel/latest.version" version_file_url="$uncached_feed/aspnetcore/Runtime/$channel/latest.version"
elif [ -z "$runtime" ]; then elif [ -z "$runtime" ]; then
if [ "$coherent" = true ]; then version_file_url="$uncached_feed/Sdk/$channel/latest.version"
version_file_url="$uncached_feed/Sdk/$channel/latest.coherent.version"
else
version_file_url="$uncached_feed/Sdk/$channel/latest.version"
fi
else else
say_err "Invalid value for \$runtime" say_err "Invalid value for \$runtime"
return 1 return 1
@@ -468,7 +453,6 @@ parse_jsonfile_for_version() {
sdk_list=$(echo $sdk_section | awk -F"[{}]" '{print $2}') sdk_list=$(echo $sdk_section | awk -F"[{}]" '{print $2}')
sdk_list=${sdk_list//[\" ]/} sdk_list=${sdk_list//[\" ]/}
sdk_list=${sdk_list//,/$'\n'} sdk_list=${sdk_list//,/$'\n'}
sdk_list="$(echo -e "${sdk_list}" | tr -d '[[:space:]]')"
local version_info="" local version_info=""
while read -r line; do while read -r line; do
@@ -505,26 +489,16 @@ get_specific_version_from_version() {
local json_file="$5" local json_file="$5"
if [ -z "$json_file" ]; then if [ -z "$json_file" ]; then
case "$version" in if [[ "$version" == "latest" ]]; then
latest) local version_info
local version_info version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1
version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" false)" || return 1 say_verbose "get_specific_version_from_version: version_info=$version_info"
say_verbose "get_specific_version_from_version: version_info=$version_info" echo "$version_info" | get_version_from_version_info
echo "$version_info" | get_version_from_version_info return 0
return 0 else
;; echo "$version"
coherent) return 0
local version_info fi
version_info="$(get_latest_version_info "$azure_feed" "$channel" "$normalized_architecture" true)" || return 1
say_verbose "get_specific_version_from_version: version_info=$version_info"
echo "$version_info" | get_version_from_version_info
return 0
;;
*)
echo "$version"
return 0
;;
esac
else else
local version_info local version_info
version_info="$(parse_jsonfile_for_version "$json_file")" || return 1 version_info="$(parse_jsonfile_for_version "$json_file")" || return 1
@@ -538,6 +512,7 @@ get_specific_version_from_version() {
# channel - $2 # channel - $2
# normalized_architecture - $3 # normalized_architecture - $3
# specific_version - $4 # specific_version - $4
# normalized_os - $5
construct_download_link() { construct_download_link() {
eval $invocation eval $invocation
@@ -545,17 +520,16 @@ construct_download_link() {
local channel="$2" local channel="$2"
local normalized_architecture="$3" local normalized_architecture="$3"
local specific_version="${4//[$'\t\r\n']}" local specific_version="${4//[$'\t\r\n']}"
local specific_product_version="$(get_specific_product_version "$1" "$4")"
local osname local osname="$5"
osname="$(get_current_os_name)" || return 1
local download_link=null local download_link=null
if [[ "$runtime" == "dotnet" ]]; then if [[ "$runtime" == "dotnet" ]]; then
download_link="$azure_feed/Runtime/$specific_version/dotnet-runtime-$specific_version-$osname-$normalized_architecture.tar.gz" download_link="$azure_feed/Runtime/$specific_version/dotnet-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz"
elif [[ "$runtime" == "aspnetcore" ]]; then elif [[ "$runtime" == "aspnetcore" ]]; then
download_link="$azure_feed/aspnetcore/Runtime/$specific_version/aspnetcore-runtime-$specific_version-$osname-$normalized_architecture.tar.gz" download_link="$azure_feed/aspnetcore/Runtime/$specific_version/aspnetcore-runtime-$specific_product_version-$osname-$normalized_architecture.tar.gz"
elif [ -z "$runtime" ]; then elif [ -z "$runtime" ]; then
download_link="$azure_feed/Sdk/$specific_version/dotnet-sdk-$specific_version-$osname-$normalized_architecture.tar.gz" download_link="$azure_feed/Sdk/$specific_version/dotnet-sdk-$specific_product_version-$osname-$normalized_architecture.tar.gz"
else else
return 1 return 1
fi fi
@@ -564,6 +538,50 @@ construct_download_link() {
return 0 return 0
} }
# args:
# azure_feed - $1
# specific_version - $2
get_specific_product_version() {
# If we find a 'productVersion.txt' at the root of any folder, we'll use its contents
# to resolve the version of what's in the folder, superseding the specified version.
eval $invocation
local azure_feed="$1"
local specific_version="${2//[$'\t\r\n']}"
local specific_product_version=$specific_version
local download_link=null
if [[ "$runtime" == "dotnet" ]]; then
download_link="$azure_feed/Runtime/$specific_version/productVersion.txt${feed_credential}"
elif [[ "$runtime" == "aspnetcore" ]]; then
download_link="$azure_feed/aspnetcore/Runtime/$specific_version/productVersion.txt${feed_credential}"
elif [ -z "$runtime" ]; then
download_link="$azure_feed/Sdk/$specific_version/productVersion.txt${feed_credential}"
else
return 1
fi
if machine_has "curl"
then
specific_product_version=$(curl -s --fail "$download_link")
if [ $? -ne 0 ]
then
specific_product_version=$specific_version
fi
elif machine_has "wget"
then
specific_product_version=$(wget -qO- "$download_link")
if [ $? -ne 0 ]
then
specific_product_version=$specific_version
fi
fi
specific_product_version="${specific_product_version//[$'\t\r\n']}"
echo "$specific_product_version"
return 0
}
# args: # args:
# azure_feed - $1 # azure_feed - $1
# channel - $2 # channel - $2
@@ -684,11 +702,31 @@ extract_dotnet_package() {
find "$temp_out_path" -type f | grep -Ev "$folders_with_version_regex" | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" "$override_non_versioned_files" find "$temp_out_path" -type f | grep -Ev "$folders_with_version_regex" | copy_files_or_dirs_from_list "$temp_out_path" "$out_path" "$override_non_versioned_files"
rm -rf "$temp_out_path" rm -rf "$temp_out_path"
rm -f "$zip_path" && say_verbose "Temporary zip file $zip_path was removed"
if [ "$failed" = true ]; then if [ "$failed" = true ]; then
say_err "Extraction failed" say_err "Extraction failed"
return 1 return 1
fi fi
return 0
}
get_http_header_curl() {
eval $invocation
local remote_path="$1"
remote_path_with_credential="${remote_path}${feed_credential}"
curl_options="-I -sSL --retry 5 --retry-delay 2 --connect-timeout 15 "
curl $curl_options "$remote_path_with_credential" || return 1
return 0
}
get_http_header_wget() {
eval $invocation
local remote_path="$1"
remote_path_with_credential="${remote_path}${feed_credential}"
wget_options="-q -S --spider --tries 5 --waitretry 2 --connect-timeout 15 "
wget $wget_options "$remote_path_with_credential" 2>&1 || return 1
return 0
} }
# args: # args:
@@ -720,44 +758,56 @@ download() {
return 0 return 0
} }
# Updates global variables $http_code and $download_error_msg
downloadcurl() { downloadcurl() {
eval $invocation eval $invocation
local remote_path="$1" local remote_path="$1"
local out_path="${2:-}" local out_path="${2:-}"
# Append feed_credential as late as possible before calling curl to avoid logging feed_credential # Append feed_credential as late as possible before calling curl to avoid logging feed_credential
remote_path="${remote_path}${feed_credential}" local remote_path_with_credential="${remote_path}${feed_credential}"
local curl_options="--retry 20 --retry-delay 2 --connect-timeout 15 -sSL -f --create-dirs " local curl_options="--retry 20 --retry-delay 2 --connect-timeout 15 -sSL -f --create-dirs "
local failed=false local failed=false
if [ -z "$out_path" ]; then if [ -z "$out_path" ]; then
curl $curl_options "$remote_path" || failed=true curl $curl_options "$remote_path_with_credential" || failed=true
else else
curl $curl_options -o "$out_path" "$remote_path" || failed=true curl $curl_options -o "$out_path" "$remote_path_with_credential" || failed=true
fi fi
if [ "$failed" = true ]; then if [ "$failed" = true ]; then
say_verbose "Curl download failed" local response=$(get_http_header_curl $remote_path_with_credential)
http_code=$( echo "$response" | awk '/^HTTP/{print $2}' | tail -1 )
download_error_msg="Unable to download $remote_path."
if [[ $http_code != 2* ]]; then
download_error_msg+=" Returned HTTP status code: $http_code."
fi
say_verbose "$download_error_msg"
return 1 return 1
fi fi
return 0 return 0
} }
# Updates global variables $http_code and $download_error_msg
downloadwget() { downloadwget() {
eval $invocation eval $invocation
local remote_path="$1" local remote_path="$1"
local out_path="${2:-}" local out_path="${2:-}"
# Append feed_credential as late as possible before calling wget to avoid logging feed_credential # Append feed_credential as late as possible before calling wget to avoid logging feed_credential
remote_path="${remote_path}${feed_credential}" local remote_path_with_credential="${remote_path}${feed_credential}"
local wget_options="--tries 20 --waitretry 2 --connect-timeout 15 " local wget_options="--tries 20 --waitretry 2 --connect-timeout 15 "
local failed=false local failed=false
if [ -z "$out_path" ]; then if [ -z "$out_path" ]; then
wget -q $wget_options -O - "$remote_path" || failed=true wget -q $wget_options -O - "$remote_path_with_credential" || failed=true
else else
wget $wget_options -O "$out_path" "$remote_path" || failed=true wget $wget_options -O "$out_path" "$remote_path_with_credential" || failed=true
fi fi
if [ "$failed" = true ]; then if [ "$failed" = true ]; then
say_verbose "Wget download failed" local response=$(get_http_header_wget $remote_path_with_credential)
http_code=$( echo "$response" | awk '/^ HTTP/{print $2}' | tail -1 )
download_error_msg="Unable to download $remote_path."
if [[ $http_code != 2* ]]; then
download_error_msg+=" Returned HTTP status code: $http_code."
fi
say_verbose "$download_error_msg"
return 1 return 1
fi fi
return 0 return 0
@@ -770,14 +820,18 @@ calculate_vars() {
normalized_architecture="$(get_normalized_architecture_from_architecture "$architecture")" normalized_architecture="$(get_normalized_architecture_from_architecture "$architecture")"
say_verbose "normalized_architecture=$normalized_architecture" say_verbose "normalized_architecture=$normalized_architecture"
normalized_os="$(get_normalized_os "$user_defined_os")"
say_verbose "normalized_os=$normalized_os"
specific_version="$(get_specific_version_from_version "$azure_feed" "$channel" "$normalized_architecture" "$version" "$json_file")" specific_version="$(get_specific_version_from_version "$azure_feed" "$channel" "$normalized_architecture" "$version" "$json_file")"
specific_product_version="$(get_specific_product_version "$azure_feed" "$specific_version")"
say_verbose "specific_version=$specific_version" say_verbose "specific_version=$specific_version"
if [ -z "$specific_version" ]; then if [ -z "$specific_version" ]; then
say_err "Could not resolve version information." say_err "Could not resolve version information."
return 1 return 1
fi fi
download_link="$(construct_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" download_link="$(construct_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version" "$normalized_os")"
say_verbose "Constructed primary named payload URL: $download_link" say_verbose "Constructed primary named payload URL: $download_link"
legacy_download_link="$(construct_legacy_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" || valid_legacy_download_link=false legacy_download_link="$(construct_legacy_download_link "$azure_feed" "$channel" "$normalized_architecture" "$specific_version")" || valid_legacy_download_link=false
@@ -822,38 +876,76 @@ install_dotnet() {
zip_path="$(mktemp "$temporary_file_template")" zip_path="$(mktemp "$temporary_file_template")"
say_verbose "Zip path: $zip_path" say_verbose "Zip path: $zip_path"
say "Downloading link: $download_link"
# Failures are normal in the non-legacy case for ultimately legacy downloads. # Failures are normal in the non-legacy case for ultimately legacy downloads.
# Do not output to stderr, since output to stderr is considered an error. # Do not output to stderr, since output to stderr is considered an error.
say "Downloading primary link $download_link"
# The download function will set variables $http_code and $download_error_msg in case of failure.
http_code=""; download_error_msg=""
download "$download_link" "$zip_path" 2>&1 || download_failed=true download "$download_link" "$zip_path" 2>&1 || download_failed=true
primary_path_http_code="$http_code"; primary_path_download_error_msg="$download_error_msg"
# if the download fails, download the legacy_download_link # if the download fails, download the legacy_download_link
if [ "$download_failed" = true ]; then if [ "$download_failed" = true ]; then
say "Cannot download: $download_link" case $primary_path_http_code in
404)
say "The resource at $download_link is not available."
;;
*)
say "$primary_path_download_error_msg"
;;
esac
rm -f "$zip_path" 2>&1 && say_verbose "Temporary zip file $zip_path was removed"
if [ "$valid_legacy_download_link" = true ]; then if [ "$valid_legacy_download_link" = true ]; then
download_failed=false download_failed=false
download_link="$legacy_download_link" download_link="$legacy_download_link"
zip_path="$(mktemp "$temporary_file_template")" zip_path="$(mktemp "$temporary_file_template")"
say_verbose "Legacy zip path: $zip_path" say_verbose "Legacy zip path: $zip_path"
say "Downloading legacy link: $download_link"
say "Downloading legacy link $download_link"
# The download function will set variables $http_code and $download_error_msg in case of failure.
http_code=""; download_error_msg=""
download "$download_link" "$zip_path" 2>&1 || download_failed=true download "$download_link" "$zip_path" 2>&1 || download_failed=true
legacy_path_http_code="$http_code"; legacy_path_download_error_msg="$download_error_msg"
if [ "$download_failed" = true ]; then if [ "$download_failed" = true ]; then
say "Cannot download: $download_link" case $legacy_path_http_code in
404)
say "The resource at $download_link is not available."
;;
*)
say "$legacy_path_download_error_msg"
;;
esac
rm -f "$zip_path" 2>&1 && say_verbose "Temporary zip file $zip_path was removed"
fi fi
fi fi
fi fi
if [ "$download_failed" = true ]; then if [ "$download_failed" = true ]; then
say_err "Could not find/download: \`$asset_name\` with version = $specific_version" if [[ "$primary_path_http_code" = "404" && ( "$valid_legacy_download_link" = false || "$legacy_path_http_code" = "404") ]]; then
say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support" say_err "Could not find \`$asset_name\` with version = $specific_version"
say_err "Refer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support"
else
say_err "Could not download: \`$asset_name\` with version = $specific_version"
# 404-NotFound is an expected response if it goes from only one of the links, do not show that error.
# If primary path is available (not 404-NotFound) then show the primary error else show the legacy error.
if [ "$primary_path_http_code" != "404" ]; then
say_err "$primary_path_download_error_msg"
return 1
fi
if [[ "$valid_legacy_download_link" = true && "$legacy_path_http_code" != "404" ]]; then
say_err "$legacy_path_download_error_msg"
return 1
fi
fi
return 1 return 1
fi fi
say "Extracting zip from $download_link" say "Extracting zip from $download_link"
extract_dotnet_package "$zip_path" "$install_root" extract_dotnet_package "$zip_path" "$install_root" || return 1
# Check if the SDK version is installed; if not, fail the installation. # Check if the SDK version is installed; if not, fail the installation.
# if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed. # if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed.
@@ -869,12 +961,14 @@ install_dotnet() {
fi fi
# Check if the standard SDK version is installed. # Check if the standard SDK version is installed.
say_verbose "Checking installation: version = $specific_version" say_verbose "Checking installation: version = $specific_product_version"
if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_version"; then if is_dotnet_package_installed "$install_root" "$asset_relative_path" "$specific_product_version"; then
return 0 return 0
fi fi
say_err "\`$asset_name\` with version = $specific_version failed to install with an unknown error." # Version verification failed. More likely something is wrong either with the downloaded content or with the verification algorithm.
say_err "Failed to verify the version of installed \`$asset_name\`.\nInstallation source: $download_link.\nInstallation location: $install_root.\nReport the bug at https://github.com/dotnet/install-scripts/issues."
say_err "\`$asset_name\` with version = $specific_product_version failed to install with an unknown error."
return 1 return 1
} }
@@ -900,6 +994,7 @@ runtime=""
runtime_id="" runtime_id=""
override_non_versioned_files=true override_non_versioned_files=true
non_dynamic_parameters="" non_dynamic_parameters=""
user_defined_os=""
while [ $# -ne 0 ] while [ $# -ne 0 ]
do do
@@ -921,6 +1016,10 @@ do
shift shift
architecture="$1" architecture="$1"
;; ;;
--os|-[Oo][SS])
shift
user_defined_os="$1"
;;
--shared-runtime|-[Ss]hared[Rr]untime) --shared-runtime|-[Ss]hared[Rr]untime)
say_warning "The --shared-runtime flag is obsolete and may be removed in a future version of this script. The recommended usage is to specify '--runtime dotnet'." say_warning "The --shared-runtime flag is obsolete and may be removed in a future version of this script. The recommended usage is to specify '--runtime dotnet'."
if [ -z "$runtime" ]; then if [ -z "$runtime" ]; then
@@ -972,6 +1071,7 @@ do
shift shift
runtime_id="$1" runtime_id="$1"
non_dynamic_parameters+=" $name "\""$1"\""" non_dynamic_parameters+=" $name "\""$1"\"""
say_warning "Use of --runtime-id is obsolete and should be limited to the versions below 2.1. To override architecture, use --architecture option instead. To override OS, use --os option instead."
;; ;;
--jsonfile|-[Jj][Ss]on[Ff]ile) --jsonfile|-[Jj][Ss]on[Ff]ile)
shift shift
@@ -1004,8 +1104,6 @@ do
echo " -Version" echo " -Version"
echo " Possible values:" echo " Possible values:"
echo " - latest - most latest build on specific channel" echo " - latest - most latest build on specific channel"
echo " - coherent - most latest coherent build on specific channel"
echo " coherent applies only to SDK downloads"
echo " - 3-part version in a format A.B.C - represents specific version of build" echo " - 3-part version in a format A.B.C - represents specific version of build"
echo " examples: 2.0.0-preview2-006120; 1.1.0" echo " examples: 2.0.0-preview2-006120; 1.1.0"
echo " -i,--install-dir <DIR> Install under specified location (see Install Location below)" echo " -i,--install-dir <DIR> Install under specified location (see Install Location below)"
@@ -1013,6 +1111,11 @@ do
echo " --architecture <ARCHITECTURE> Architecture of dotnet binaries to be installed, Defaults to \`$architecture\`." echo " --architecture <ARCHITECTURE> Architecture of dotnet binaries to be installed, Defaults to \`$architecture\`."
echo " --arch,-Architecture,-Arch" echo " --arch,-Architecture,-Arch"
echo " Possible values: x64, arm, and arm64" echo " Possible values: x64, arm, and arm64"
echo " --os <system> Specifies operating system to be used when selecting the installer."
echo " Overrides the OS determination approach used by the script. Supported values: osx, linux, linux-musl, freebsd, rhel.6."
echo " In case any other value is provided, the platform will be determined by the script based on machine configuration."
echo " Not supported for legacy links. Use --runtime-id to specify platform for legacy links."
echo " Refer to: https://aka.ms/dotnet-os-lifecycle for more information."
echo " --runtime <RUNTIME> Installs a shared runtime only, without the SDK." echo " --runtime <RUNTIME> Installs a shared runtime only, without the SDK."
echo " -Runtime" echo " -Runtime"
echo " Possible values:" echo " Possible values:"
@@ -1029,14 +1132,15 @@ do
echo " --no-cdn,-NoCdn Disable downloading from the Azure CDN, and use the uncached feed directly." echo " --no-cdn,-NoCdn Disable downloading from the Azure CDN, and use the uncached feed directly."
echo " --jsonfile <JSONFILE> Determines the SDK version from a user specified global.json file." echo " --jsonfile <JSONFILE> Determines the SDK version from a user specified global.json file."
echo " Note: global.json must have a value for 'SDK:Version'" echo " Note: global.json must have a value for 'SDK:Version'"
echo " --runtime-id Installs the .NET Tools for the given platform (use linux-x64 for portable linux)."
echo " -RuntimeId"
echo " -?,--?,-h,--help,-Help Shows this help message" echo " -?,--?,-h,--help,-Help Shows this help message"
echo "" echo ""
echo "Obsolete parameters:" echo "Obsolete parameters:"
echo " --shared-runtime The recommended alternative is '--runtime dotnet'." echo " --shared-runtime The recommended alternative is '--runtime dotnet'."
echo " This parameter is obsolete and may be removed in a future version of this script." echo " This parameter is obsolete and may be removed in a future version of this script."
echo " Installs just the shared runtime bits, not the entire SDK." echo " Installs just the shared runtime bits, not the entire SDK."
echo " --runtime-id Installs the .NET Tools for the given platform (use linux-x64 for portable linux)."
echo " -RuntimeId" The parameter is obsolete and may be removed in a future version of this script. Should be used only for versions below 2.1.
echo " For primary links to override OS or/and architecture, use --os and --architecture option instead."
echo "" echo ""
echo "Install Location:" echo "Install Location:"
echo " Location is chosen in following order:" echo " Location is chosen in following order:"
@@ -1058,6 +1162,11 @@ if [ "$no_cdn" = true ]; then
azure_feed="$uncached_feed" azure_feed="$uncached_feed"
fi fi
say "Note that the intended use of this script is for Continuous Integration (CI) scenarios, where:"
say "- The SDK needs to be installed without user interaction and without admin rights."
say "- The SDK installation doesn't need to persist across multiple CI runs."
say "To set up a development environment or to run apps, use installers rather than this script. Visit https://dotnet.microsoft.com/download to get the installer.\n"
check_min_reqs check_min_reqs
calculate_vars calculate_vars
script_name=$(basename "$0") script_name=$(basename "$0")
@@ -1068,7 +1177,7 @@ if [ "$dry_run" = true ]; then
if [ "$valid_legacy_download_link" = true ]; then if [ "$valid_legacy_download_link" = true ]; then
say "Legacy named payload URL: $legacy_download_link" say "Legacy named payload URL: $legacy_download_link"
fi fi
repeatable_command="./$script_name --version "\""$specific_version"\"" --install-dir "\""$install_root"\"" --architecture "\""$normalized_architecture"\""" repeatable_command="./$script_name --version "\""$specific_version"\"" --install-dir "\""$install_root"\"" --architecture "\""$normalized_architecture"\"" --os "\""$normalized_os"\"""
if [[ "$runtime" == "dotnet" ]]; then if [[ "$runtime" == "dotnet" ]]; then
repeatable_command+=" --runtime "\""dotnet"\""" repeatable_command+=" --runtime "\""dotnet"\"""
elif [[ "$runtime" == "aspnetcore" ]]; then elif [[ "$runtime" == "aspnetcore" ]]; then
@@ -1079,7 +1188,6 @@ if [ "$dry_run" = true ]; then
exit 0 exit 0
fi fi
check_pre_reqs
install_dotnet install_dotnet
bin_path="$(get_absolute_path "$(combine_paths "$install_root" "$bin_folder_relative_path")")" bin_path="$(get_absolute_path "$(combine_paths "$install_root" "$bin_folder_relative_path")")"
@@ -1090,4 +1198,6 @@ else
say "Binaries of dotnet can be found in $bin_path" say "Binaries of dotnet can be found in $bin_path"
fi fi
say "Note that the script does not resolve dependencies during installation."
say "To check the list of dependencies, go to https://docs.microsoft.com/dotnet/core/install, select your operating system and check the \"Dependencies\" section."
say "Installation finished successfully." say "Installation finished successfully."

View File

@@ -5,9 +5,9 @@
"requires": true, "requires": true,
"dependencies": { "dependencies": {
"@actions/core": { "@actions/core": {
"version": "1.2.0", "version": "1.2.6",
"resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.0.tgz", "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.6.tgz",
"integrity": "sha512-ZKdyhlSlyz38S6YFfPnyNgCDZuAF2T0Qv5eHflNWytPS8Qjvz39bZFMry9Bb/dpSnqWcNeav5yM2CTYpJeY+Dw==" "integrity": "sha512-ZQYitnqiyBc3D+k7LsgSBmMDVkOVidaagDG7j3fOym77jNunWRuYx7VSHa9GNfFZh+zh61xsCjRj4JxMZlDqTA=="
}, },
"@actions/glob": { "@actions/glob": {
"version": "0.1.0", "version": "0.1.0",
@@ -1683,9 +1683,9 @@
} }
}, },
"lodash": { "lodash": {
"version": "4.17.15", "version": "4.17.19",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==",
"dev": true "dev": true
}, },
"lodash.unescape": { "lodash.unescape": {

View File

@@ -23,5 +23,7 @@
<key>ACTIONS_RUNNER_SVC</key> <key>ACTIONS_RUNNER_SVC</key>
<string>1</string> <string>1</string>
</dict> </dict>
<key>ProcessType</key>
<string>Interactive</string>
</dict> </dict>
</plist> </plist>

View File

@@ -0,0 +1,115 @@
const https = require('https')
const fs = require('fs')
const http = require('http')
const hostname = process.env['HOSTNAME'] || ''
const port = process.env['PORT'] || ''
const path = process.env['PATH'] || ''
const pat = process.env['PAT'] || ''
const proxyHost = process.env['PROXYHOST'] || ''
const proxyPort = process.env['PROXYPORT'] || ''
const proxyUsername = process.env['PROXYUSERNAME'] || ''
const proxyPassword = process.env['PROXYPASSWORD'] || ''
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0'
if (proxyHost === '') {
const options = {
hostname: hostname,
port: port,
path: path,
method: 'GET',
headers: {
'User-Agent': 'GitHubActionsRunnerCheck/1.0',
'Authorization': `token ${pat}`
},
}
const req = https.request(options, res => {
console.log(`statusCode: ${res.statusCode}`)
console.log(`headers: ${JSON.stringify(res.headers)}`)
let cert = socket.getPeerCertificate(true)
let certPEM = ''
let fingerprints = {}
while (cert != null && fingerprints[cert.fingerprint] != '1') {
fingerprints[cert.fingerprint] = '1'
certPEM = certPEM + '-----BEGIN CERTIFICATE-----\n'
let certEncoded = cert.raw.toString('base64')
for (let i = 0; i < certEncoded.length; i++) {
certPEM = certPEM + certEncoded[i]
if (i != certEncoded.length - 1 && (i + 1) % 64 == 0) {
certPEM = certPEM + '\n'
}
}
certPEM = certPEM + '\n-----END CERTIFICATE-----\n'
cert = cert.issuerCertificate
}
console.log(certPEM)
fs.writeFileSync('./download_ca_cert.pem', certPEM)
res.on('data', d => {
process.stdout.write(d)
})
})
req.on('error', error => {
console.error(error)
})
req.end()
}
else {
const auth = 'Basic ' + Buffer.from(proxyUsername + ':' + proxyPassword).toString('base64')
const options = {
host: proxyHost,
port: proxyPort,
method: 'CONNECT',
path: `${hostname}:${port}`,
}
if (proxyUsername != '' || proxyPassword != '') {
options.headers = {
'Proxy-Authorization': auth,
}
}
http.request(options).on('connect', (res, socket) => {
if (res.statusCode != 200) {
throw new Error(`Proxy returns code: ${res.statusCode}`)
}
https.get({
host: hostname,
port: port,
socket: socket,
agent: false,
path: '/',
headers: {
'User-Agent': 'GitHubActionsRunnerCheck/1.0',
'Authorization': `token ${pat}`
}
}, (res) => {
let cert = res.socket.getPeerCertificate(true)
let certPEM = ''
let fingerprints = {}
while (cert != null && fingerprints[cert.fingerprint] != '1') {
fingerprints[cert.fingerprint] = '1'
certPEM = certPEM + '-----BEGIN CERTIFICATE-----\n'
let certEncoded = cert.raw.toString('base64')
for (let i = 0; i < certEncoded.length; i++) {
certPEM = certPEM + certEncoded[i]
if (i != certEncoded.length - 1 && (i + 1) % 64 == 0) {
certPEM = certPEM + '\n'
}
}
certPEM = certPEM + '\n-----END CERTIFICATE-----\n'
cert = cert.issuerCertificate
}
console.log(certPEM)
fs.writeFileSync('./download_ca_cert.pem', certPEM)
console.log(`statusCode: ${res.statusCode}`)
console.log(`headers: ${JSON.stringify(res.headers)}`)
res.on('data', d => {
process.stdout.write(d)
})
})
}).on('error', (err) => {
console.error('error', err)
}).end()
}

View File

@@ -0,0 +1,75 @@
const https = require('https')
const http = require('http')
const hostname = process.env['HOSTNAME'] || ''
const port = process.env['PORT'] || ''
const path = process.env['PATH'] || ''
const pat = process.env['PAT'] || ''
const proxyHost = process.env['PROXYHOST'] || ''
const proxyPort = process.env['PROXYPORT'] || ''
const proxyUsername = process.env['PROXYUSERNAME'] || ''
const proxyPassword = process.env['PROXYPASSWORD'] || ''
if (proxyHost === '') {
const options = {
hostname: hostname,
port: port,
path: path,
method: 'GET',
headers: {
'User-Agent': 'GitHubActionsRunnerCheck/1.0',
'Authorization': `token ${pat}`,
}
}
const req = https.request(options, res => {
console.log(`statusCode: ${res.statusCode}`)
console.log(`headers: ${JSON.stringify(res.headers)}`)
res.on('data', d => {
process.stdout.write(d)
})
})
req.on('error', error => {
console.error(error)
})
req.end()
}
else {
const proxyAuth = 'Basic ' + Buffer.from(proxyUsername + ':' + proxyPassword).toString('base64')
const options = {
hostname: proxyHost,
port: proxyPort,
method: 'CONNECT',
path: `${hostname}:${port}`
}
if (proxyUsername != '' || proxyPassword != '') {
options.headers = {
'Proxy-Authorization': proxyAuth,
}
}
http.request(options).on('connect', (res, socket) => {
if (res.statusCode != 200) {
throw new Error(`Proxy returns code: ${res.statusCode}`)
}
https.get({
host: hostname,
port: port,
socket: socket,
agent: false,
path: path,
headers: {
'User-Agent': 'GitHubActionsRunnerCheck/1.0',
'Authorization': `token ${pat}`,
}
}, (res) => {
console.log(`statusCode: ${res.statusCode}`)
console.log(`headers: ${JSON.stringify(res.headers)}`)
res.on('data', d => {
process.stdout.write(d)
})
})
}).on('error', (err) => {
console.error('error', err)
}).end()
}

View File

@@ -1,6 +1,7 @@
#!/bin/bash #!/bin/bash
SVC_NAME="{{SvcNameVar}}" SVC_NAME="{{SvcNameVar}}"
SVC_NAME=${SVC_NAME// /_}
SVC_DESCRIPTION="{{SvcDescription}}" SVC_DESCRIPTION="{{SvcDescription}}"
user_id=`id -u` user_id=`id -u`

View File

@@ -14,14 +14,14 @@ fi
function print_errormessage() function print_errormessage()
{ {
echo "Can't install dotnet core dependencies." echo "Can't install dotnet 5 dependencies."
echo "You can manually install all required dependencies based on following documentation" echo "You can manually install all required dependencies based on following documentation"
echo "https://docs.microsoft.com/en-us/dotnet/core/linux-prerequisites?tabs=netcore2x" echo "https://docs.microsoft.com/en-us/dotnet/core/linux-prerequisites?tabs=netcore2x"
} }
function print_rhel6message() function print_rhel6message()
{ {
echo "We did our best effort to install dotnet core dependencies" echo "We did our best effort to install dotnet 5 dependencies"
echo "However, there are some dependencies which require manual installation" echo "However, there are some dependencies which require manual installation"
echo "You can install all remaining required dependencies based on the following documentation" echo "You can install all remaining required dependencies based on the following documentation"
echo "https://github.com/dotnet/core/blob/master/Documentation/build-and-install-rhel6-prerequisites.md" echo "https://github.com/dotnet/core/blob/master/Documentation/build-and-install-rhel6-prerequisites.md"
@@ -29,7 +29,7 @@ function print_rhel6message()
function print_rhel6errormessage() function print_rhel6errormessage()
{ {
echo "We couldn't install dotnet core dependencies" echo "We couldn't install dotnet 5 dependencies"
echo "You can manually install all required dependencies based on following documentation" echo "You can manually install all required dependencies based on following documentation"
echo "https://docs.microsoft.com/en-us/dotnet/core/linux-prerequisites?tabs=netcore2x" echo "https://docs.microsoft.com/en-us/dotnet/core/linux-prerequisites?tabs=netcore2x"
echo "In addition, there are some dependencies which require manual installation. Please follow this documentation" echo "In addition, there are some dependencies which require manual installation. Please follow this documentation"
@@ -49,70 +49,68 @@ then
cat /etc/debian_version cat /etc/debian_version
echo "------------------------------" echo "------------------------------"
# prefer apt over apt-get # prefer apt-get over apt
command -v apt command -v apt-get
if [ $? -eq 0 ] if [ $? -eq 0 ]
then then
apt update && apt install -y liblttng-ust0 libkrb5-3 zlib1g apt_get=apt-get
if [ $? -ne 0 ]
then
echo "'apt' failed with exit code '$?'"
print_errormessage
exit 1
fi
# libissl version prefer: libssl1.1 -> libssl1.0.2 -> libssl1.0.0
apt install -y libssl1.1$ || apt install -y libssl1.0.2$ || apt install -y libssl1.0.0$
if [ $? -ne 0 ]
then
echo "'apt' failed with exit code '$?'"
print_errormessage
exit 1
fi
# libicu version prefer: libicu63 -> libicu60 -> libicu57 -> libicu55 -> libicu52
apt install -y libicu63 || apt install -y libicu60 || apt install -y libicu57 || apt install -y libicu55 || apt install -y libicu52
if [ $? -ne 0 ]
then
echo "'apt' failed with exit code '$?'"
print_errormessage
exit 1
fi
else else
command -v apt-get command -v apt
if [ $? -eq 0 ] if [ $? -eq 0 ]
then then
apt-get update && apt-get install -y liblttng-ust0 libkrb5-3 zlib1g apt_get=apt
if [ $? -ne 0 ]
then
echo "'apt-get' failed with exit code '$?'"
print_errormessage
exit 1
fi
# libissl version prefer: libssl1.1 -> libssl1.0.2 -> libssl1.0.0
apt-get install -y libssl1.1$ || apt-get install -y libssl1.0.2$ || apt install -y libssl1.0.0$
if [ $? -ne 0 ]
then
echo "'apt-get' failed with exit code '$?'"
print_errormessage
exit 1
fi
# libicu version prefer: libicu63 -> libicu60 -> libicu57 -> libicu55 -> libicu52
apt-get install -y libicu63 || apt-get install -y libicu60 || apt install -y libicu57 || apt install -y libicu55 || apt install -y libicu52
if [ $? -ne 0 ]
then
echo "'apt-get' failed with exit code '$?'"
print_errormessage
exit 1
fi
else else
echo "Can not find 'apt' or 'apt-get'" echo "Found neither 'apt-get' nor 'apt'"
print_errormessage print_errormessage
exit 1 exit 1
fi fi
fi fi
$apt_get update && $apt_get install -y liblttng-ust0 libkrb5-3 zlib1g
if [ $? -ne 0 ]
then
echo "'$apt_get' failed with exit code '$?'"
print_errormessage
exit 1
fi
apt_get_with_fallbacks() {
$apt_get install -y $1
fail=$?
if [ $fail -eq 0 ]
then
if [ "${1#"${1%?}"}" = '$' ]; then
dpkg -l "${1%?}" > /dev/null 2> /dev/null
fail=$?
fi
fi
if [ $fail -ne 0 ]
then
shift
if [ -n "$1" ]
then
apt_get_with_fallbacks "$@"
fi
fi
}
# libssl version prefer: libssl1.1 -> libssl1.0.2 -> libssl1.0.0
apt_get_with_fallbacks libssl1.1$ libssl1.0.2$ libssl1.0.0$
if [ $? -ne 0 ]
then
echo "'$apt_get' failed with exit code '$?'"
print_errormessage
exit 1
fi
# libicu version prefer: libicu66 -> libicu63 -> libicu60 -> libicu57 -> libicu55 -> libicu52
apt_get_with_fallbacks libicu66 libicu63 libicu60 libicu57 libicu55 libicu52
if [ $? -ne 0 ]
then
echo "'$apt_get' failed with exit code '$?'"
print_errormessage
exit 1
fi
elif [ -e /etc/redhat-release ] elif [ -e /etc/redhat-release ]
then then
echo "The current OS is Fedora based" echo "The current OS is Fedora based"

View File

@@ -1,6 +1,7 @@
#!/bin/bash #!/bin/bash
SVC_NAME="{{SvcNameVar}}" SVC_NAME="{{SvcNameVar}}"
SVC_NAME=${SVC_NAME// /_}
SVC_DESCRIPTION="{{SvcDescription}}" SVC_DESCRIPTION="{{SvcDescription}}"
SVC_CMD=$1 SVC_CMD=$1
@@ -62,12 +63,25 @@ function install()
sed "s/{{User}}/${run_as_user}/g; s/{{Description}}/$(echo ${SVC_DESCRIPTION} | sed -e 's/[\/&]/\\&/g')/g; s/{{RunnerRoot}}/$(echo ${RUNNER_ROOT} | sed -e 's/[\/&]/\\&/g')/g;" "${TEMPLATE_PATH}" > "${TEMP_PATH}" || failed "failed to create replacement temp file" sed "s/{{User}}/${run_as_user}/g; s/{{Description}}/$(echo ${SVC_DESCRIPTION} | sed -e 's/[\/&]/\\&/g')/g; s/{{RunnerRoot}}/$(echo ${RUNNER_ROOT} | sed -e 's/[\/&]/\\&/g')/g;" "${TEMPLATE_PATH}" > "${TEMP_PATH}" || failed "failed to create replacement temp file"
mv "${TEMP_PATH}" "${UNIT_PATH}" || failed "failed to copy unit file" mv "${TEMP_PATH}" "${UNIT_PATH}" || failed "failed to copy unit file"
# Recent Fedora based Linux (CentOS/Redhat) has SELinux enabled by default
# We need to restore security context on the unit file we added otherwise SystemD have no access to it.
command -v getenforce > /dev/null
if [ $? -eq 0 ]
then
selinuxEnabled=$(getenforce)
if [[ $selinuxEnabled == "Enforcing" ]]
then
# SELinux is enabled, we will need to Restore SELinux Context for the service file
restorecon -r -v "${UNIT_PATH}" || failed "failed to restore SELinux context on ${UNIT_PATH}"
fi
fi
# unit file should not be executable and world writable # unit file should not be executable and world writable
chmod 664 ${UNIT_PATH} || failed "failed to set permissions on ${UNIT_PATH}" chmod 664 "${UNIT_PATH}" || failed "failed to set permissions on ${UNIT_PATH}"
systemctl daemon-reload || failed "failed to reload daemons" systemctl daemon-reload || failed "failed to reload daemons"
# Since we started with sudo, runsvc.sh will be owned by root. Change this to current login user. # Since we started with sudo, runsvc.sh will be owned by root. Change this to current login user.
cp ./bin/runsvc.sh ./runsvc.sh || failed "failed to copy runsvc.sh" cp ./bin/runsvc.sh ./runsvc.sh || failed "failed to copy runsvc.sh"
chown ${run_as_uid}:${run_as_gid} ./runsvc.sh || failed "failed to set owner for runsvc.sh" chown ${run_as_uid}:${run_as_gid} ./runsvc.sh || failed "failed to set owner for runsvc.sh"
chmod 755 ./runsvc.sh || failed "failed to set permission for runsvc.sh" chmod 755 ./runsvc.sh || failed "failed to set permission for runsvc.sh"

4
src/Misc/layoutbin/update.sh.template Normal file → Executable file
View File

@@ -28,13 +28,13 @@ date "+[%F %T-%4N] Waiting for $runnerprocessname ($runnerpid) to complete" >> "
while [ -e /proc/$runnerpid ] while [ -e /proc/$runnerpid ]
do do
date "+[%F %T-%4N] Process $runnerpid still running" >> "$logfile" 2>&1 date "+[%F %T-%4N] Process $runnerpid still running" >> "$logfile" 2>&1
ping -c 2 127.0.0.1 >nul sleep 2
done done
date "+[%F %T-%4N] Process $runnerpid finished running" >> "$logfile" 2>&1 date "+[%F %T-%4N] Process $runnerpid finished running" >> "$logfile" 2>&1
# start re-organize folders # start re-organize folders
date "+[%F %T-%4N] Sleep 1 more second to make sure process exited" >> "$logfile" 2>&1 date "+[%F %T-%4N] Sleep 1 more second to make sure process exited" >> "$logfile" 2>&1
ping -c 2 127.0.0.1 >nul sleep 1
# the folder structure under runner root will be # the folder structure under runner root will be
# ./bin -> bin.2.100.0 (junction folder) # ./bin -> bin.2.100.0 (junction folder)

View File

@@ -8,7 +8,7 @@ if [ $user_id -eq 0 -a -z "$RUNNER_ALLOW_RUNASROOT" ]; then
exit 1 exit 1
fi fi
# Check dotnet core 3.0 dependencies for Linux # Check dotnet 5 dependencies for Linux
if [[ (`uname` == "Linux") ]] if [[ (`uname` == "Linux") ]]
then then
command -v ldd > /dev/null command -v ldd > /dev/null
@@ -18,24 +18,26 @@ then
exit 1 exit 1
fi fi
message="Execute sudo ./bin/installdependencies.sh to install any missing Dotnet 5 dependencies."
ldd ./bin/libcoreclr.so | grep 'not found' ldd ./bin/libcoreclr.so | grep 'not found'
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
echo "Dependencies is missing for Dotnet Core 3.0" echo "Dependencies is missing for Dotnet 5"
echo "Execute ./bin/installdependencies.sh to install any missing Dotnet Core 3.0 dependencies." echo $message
exit 1 exit 1
fi fi
ldd ./bin/System.Security.Cryptography.Native.OpenSsl.so | grep 'not found' ldd ./bin/libSystem.Security.Cryptography.Native.OpenSsl.so | grep 'not found'
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
echo "Dependencies is missing for Dotnet Core 3.0" echo "Dependencies is missing for Dotnet 5"
echo "Execute ./bin/installdependencies.sh to install any missing Dotnet Core 3.0 dependencies." echo $message
exit 1 exit 1
fi fi
ldd ./bin/System.IO.Compression.Native.so | grep 'not found' ldd ./bin/libSystem.IO.Compression.Native.so | grep 'not found'
if [ $? -eq 0 ]; then if [ $? -eq 0 ]; then
echo "Dependencies is missing for Dotnet Core 3.0" echo "Dependencies is missing for Dotnet 5"
echo "Execute ./bin/installdependencies.sh to install any missing Dotnet Core 3.0 dependencies." echo $message
exit 1 exit 1
fi fi
@@ -50,10 +52,10 @@ then
fi fi
libpath=${LD_LIBRARY_PATH:-} libpath=${LD_LIBRARY_PATH:-}
$LDCONFIG_COMMAND -NXv ${libpath//:/} 2>&1 | grep libicu >/dev/null 2>&1 $LDCONFIG_COMMAND -NXv ${libpath//:/ } 2>&1 | grep libicu >/dev/null 2>&1
if [ $? -ne 0 ]; then if [ $? -ne 0 ]; then
echo "Libicu's dependencies is missing for Dotnet Core 3.0" echo "Libicu's dependencies is missing for Dotnet 5"
echo "Execute ./bin/installdependencies.sh to install any missing Dotnet Core 3.0 dependencies." echo $message
exit 1 exit 1
fi fi
fi fi
@@ -67,7 +69,7 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done done
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
cd $DIR cd "$DIR"
source ./env.sh source ./env.sh

View File

@@ -6,6 +6,7 @@ varCheckList=(
'ANT_HOME' 'ANT_HOME'
'M2_HOME' 'M2_HOME'
'ANDROID_HOME' 'ANDROID_HOME'
'ANDROID_SDK_ROOT'
'GRADLE_HOME' 'GRADLE_HOME'
'NVM_BIN' 'NVM_BIN'
'NVM_PATH' 'NVM_PATH'

View File

@@ -108,9 +108,9 @@ namespace GitHub.Runner.Common
CredentialData GetMigratedCredentials(); CredentialData GetMigratedCredentials();
RunnerSettings GetSettings(); RunnerSettings GetSettings();
void SaveCredential(CredentialData credential); void SaveCredential(CredentialData credential);
void SaveMigratedCredential(CredentialData credential);
void SaveSettings(RunnerSettings settings); void SaveSettings(RunnerSettings settings);
void DeleteCredential(); void DeleteCredential();
void DeleteMigratedCredential();
void DeleteSettings(); void DeleteSettings();
} }
@@ -232,21 +232,6 @@ namespace GitHub.Runner.Common
File.SetAttributes(_credFilePath, File.GetAttributes(_credFilePath) | FileAttributes.Hidden); File.SetAttributes(_credFilePath, File.GetAttributes(_credFilePath) | FileAttributes.Hidden);
} }
public void SaveMigratedCredential(CredentialData credential)
{
Trace.Info("Saving {0} migrated credential @ {1}", credential.Scheme, _migratedCredFilePath);
if (File.Exists(_migratedCredFilePath))
{
// Delete existing credential file first, since the file is hidden and not able to overwrite.
Trace.Info("Delete exist runner migrated credential file.");
IOUtil.DeleteFile(_migratedCredFilePath);
}
IOUtil.SaveObject(credential, _migratedCredFilePath);
Trace.Info("Migrated Credentials Saved.");
File.SetAttributes(_migratedCredFilePath, File.GetAttributes(_migratedCredFilePath) | FileAttributes.Hidden);
}
public void SaveSettings(RunnerSettings settings) public void SaveSettings(RunnerSettings settings)
{ {
Trace.Info("Saving runner settings."); Trace.Info("Saving runner settings.");
@@ -268,6 +253,11 @@ namespace GitHub.Runner.Common
IOUtil.Delete(_migratedCredFilePath, default(CancellationToken)); IOUtil.Delete(_migratedCredFilePath, default(CancellationToken));
} }
public void DeleteMigratedCredential()
{
IOUtil.Delete(_migratedCredFilePath, default(CancellationToken));
}
public void DeleteSettings() public void DeleteSettings()
{ {
IOUtil.Delete(_configFilePath, default(CancellationToken)); IOUtil.Delete(_configFilePath, default(CancellationToken));

View File

@@ -90,7 +90,7 @@ namespace GitHub.Runner.Common
public static readonly string Labels = "labels"; public static readonly string Labels = "labels";
public static readonly string MonitorSocketAddress = "monitorsocketaddress"; public static readonly string MonitorSocketAddress = "monitorsocketaddress";
public static readonly string Name = "name"; public static readonly string Name = "name";
public static readonly string Pool = "pool"; public static readonly string RunnerGroup = "runnergroup";
public static readonly string StartupType = "startuptype"; public static readonly string StartupType = "startuptype";
public static readonly string Url = "url"; public static readonly string Url = "url";
public static readonly string UserName = "username"; public static readonly string UserName = "username";
@@ -99,9 +99,11 @@ namespace GitHub.Runner.Common
// Secret args. Must be added to the "Secrets" getter as well. // Secret args. Must be added to the "Secrets" getter as well.
public static readonly string Token = "token"; public static readonly string Token = "token";
public static readonly string PAT = "pat";
public static readonly string WindowsLogonPassword = "windowslogonpassword"; public static readonly string WindowsLogonPassword = "windowslogonpassword";
public static string[] Secrets => new[] public static string[] Secrets => new[]
{ {
PAT,
Token, Token,
WindowsLogonPassword, WindowsLogonPassword,
}; };
@@ -119,6 +121,7 @@ namespace GitHub.Runner.Common
//validFlags array as well present in the CommandSettings.cs //validFlags array as well present in the CommandSettings.cs
public static class Flags public static class Flags
{ {
public static readonly string Check = "check";
public static readonly string Commit = "commit"; public static readonly string Commit = "commit";
public static readonly string Help = "help"; public static readonly string Help = "help";
public static readonly string Replace = "replace"; public static readonly string Replace = "replace";
@@ -138,8 +141,16 @@ namespace GitHub.Runner.Common
public const int RunOnceRunnerUpdating = 4; public const int RunOnceRunnerUpdating = 4;
} }
public static class Features
{
public static readonly string DiskSpaceWarning = "runner.diskspace.warning";
}
public static readonly string InternalTelemetryIssueDataKey = "_internal_telemetry"; public static readonly string InternalTelemetryIssueDataKey = "_internal_telemetry";
public static readonly string WorkerCrash = "WORKER_CRASH"; public static readonly string WorkerCrash = "WORKER_CRASH";
public static readonly string LowDiskSpace = "LOW_DISK_SPACE";
public static readonly string UnsupportedCommand = "UNSUPPORTED_COMMAND";
public static readonly string UnsupportedCommandMessageDisabled = "The `{0}` command is disabled. Please upgrade to using Environment Files or opt into unsecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_COMMANDS` environment variable to `true`. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/";
} }
public static class RunnerEvent public static class RunnerEvent
@@ -198,6 +209,7 @@ namespace GitHub.Runner.Common
// //
// Keep alphabetical // Keep alphabetical
// //
public static readonly string AllowUnsupportedCommands = "ACTIONS_ALLOW_UNSECURE_COMMANDS";
public static readonly string RunnerDebug = "ACTIONS_RUNNER_DEBUG"; public static readonly string RunnerDebug = "ACTIONS_RUNNER_DEBUG";
public static readonly string StepDebug = "ACTIONS_STEP_DEBUG"; public static readonly string StepDebug = "ACTIONS_STEP_DEBUG";
} }

View File

@@ -56,6 +56,16 @@ namespace GitHub.Runner.Common
Add<T>(extensions, "GitHub.Runner.Worker.EndGroupCommandExtension, Runner.Worker"); Add<T>(extensions, "GitHub.Runner.Worker.EndGroupCommandExtension, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.EchoCommandExtension, Runner.Worker"); Add<T>(extensions, "GitHub.Runner.Worker.EchoCommandExtension, Runner.Worker");
break; break;
case "GitHub.Runner.Worker.IFileCommandExtension":
Add<T>(extensions, "GitHub.Runner.Worker.AddPathFileCommand, Runner.Worker");
Add<T>(extensions, "GitHub.Runner.Worker.SetEnvFileCommand, Runner.Worker");
break;
case "GitHub.Runner.Listener.Check.ICheckExtension":
Add<T>(extensions, "GitHub.Runner.Listener.Check.InternetCheck, Runner.Listener");
Add<T>(extensions, "GitHub.Runner.Listener.Check.ActionsCheck, Runner.Listener");
Add<T>(extensions, "GitHub.Runner.Listener.Check.GitCheck, Runner.Listener");
Add<T>(extensions, "GitHub.Runner.Listener.Check.NodeJsCheck, Runner.Listener");
break;
default: default:
// This should never happen. // This should never happen.
throw new NotSupportedException($"Unexpected extension type: '{typeof(T).FullName}'"); throw new NotSupportedException($"Unexpected extension type: '{typeof(T).FullName}'");

View File

@@ -16,12 +16,14 @@ namespace GitHub.Runner.Common
// logging and console // logging and console
Task<TaskLog> AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken); Task<TaskLog> AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken);
Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, CancellationToken cancellationToken); Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, CancellationToken cancellationToken);
Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long startLine, CancellationToken cancellationToken);
Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken); Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken);
Task<TaskLog> CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken); Task<TaskLog> CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken);
Task<Timeline> CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken); Task<Timeline> CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken);
Task<List<TimelineRecord>> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable<TimelineRecord> records, CancellationToken cancellationToken); Task<List<TimelineRecord>> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable<TimelineRecord> records, CancellationToken cancellationToken);
Task RaisePlanEventAsync<T>(Guid scopeIdentifier, string hubName, Guid planId, T eventData, CancellationToken cancellationToken) where T : JobEvent; Task RaisePlanEventAsync<T>(Guid scopeIdentifier, string hubName, Guid planId, T eventData, CancellationToken cancellationToken) where T : JobEvent;
Task<Timeline> GetTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken); Task<Timeline> GetTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken);
Task<ActionDownloadInfoCollection> ResolveActionDownloadInfoAsync(Guid scopeIdentifier, string hubName, Guid planId, ActionReferenceList actions, CancellationToken cancellationToken);
} }
public sealed class JobServer : RunnerService, IJobServer public sealed class JobServer : RunnerService, IJobServer
@@ -78,6 +80,12 @@ namespace GitHub.Runner.Common
return _taskClient.AppendTimelineRecordFeedAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, stepId, lines, cancellationToken: cancellationToken); return _taskClient.AppendTimelineRecordFeedAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, stepId, lines, cancellationToken: cancellationToken);
} }
public Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList<string> lines, long startLine, CancellationToken cancellationToken)
{
CheckConnection();
return _taskClient.AppendTimelineRecordFeedAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, stepId, lines, startLine, cancellationToken: cancellationToken);
}
public Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, string type, string name, Stream uploadStream, CancellationToken cancellationToken) public Task<TaskAttachment> CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, string type, string name, Stream uploadStream, CancellationToken cancellationToken)
{ {
CheckConnection(); CheckConnection();
@@ -113,5 +121,14 @@ namespace GitHub.Runner.Common
CheckConnection(); CheckConnection();
return _taskClient.GetTimelineAsync(scopeIdentifier, hubName, planId, timelineId, includeRecords: true, cancellationToken: cancellationToken); return _taskClient.GetTimelineAsync(scopeIdentifier, hubName, planId, timelineId, includeRecords: true, cancellationToken: cancellationToken);
} }
//-----------------------------------------------------------------
// Action download info
//-----------------------------------------------------------------
public Task<ActionDownloadInfoCollection> ResolveActionDownloadInfoAsync(Guid scopeIdentifier, string hubName, Guid planId, ActionReferenceList actions, CancellationToken cancellationToken)
{
CheckConnection();
return _taskClient.ResolveActionDownloadInfoAsync(scopeIdentifier, hubName, planId, actions, cancellationToken: cancellationToken);
}
} }
} }

View File

@@ -18,7 +18,7 @@ namespace GitHub.Runner.Common
event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling; event EventHandler<ThrottlingEventArgs> JobServerQueueThrottling;
Task ShutdownAsync(); Task ShutdownAsync();
void Start(Pipelines.AgentJobRequestMessage jobRequest); void Start(Pipelines.AgentJobRequestMessage jobRequest);
void QueueWebConsoleLine(Guid stepRecordId, string line); void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null);
void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource); void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource);
void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord); void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord);
} }
@@ -155,10 +155,10 @@ namespace GitHub.Runner.Common
Trace.Info("All queue process tasks have been stopped, and all queues are drained."); Trace.Info("All queue process tasks have been stopped, and all queues are drained.");
} }
public void QueueWebConsoleLine(Guid stepRecordId, string line) public void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber)
{ {
Trace.Verbose("Enqueue web console line queue: {0}", line); Trace.Verbose("Enqueue web console line queue: {0}", line);
_webConsoleLineQueue.Enqueue(new ConsoleLineInfo(stepRecordId, line)); _webConsoleLineQueue.Enqueue(new ConsoleLineInfo(stepRecordId, line, lineNumber));
} }
public void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource) public void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource)
@@ -214,7 +214,7 @@ namespace GitHub.Runner.Common
} }
// Group consolelines by timeline record of each step // Group consolelines by timeline record of each step
Dictionary<Guid, List<string>> stepsConsoleLines = new Dictionary<Guid, List<string>>(); Dictionary<Guid, List<TimelineRecordLogLine>> stepsConsoleLines = new Dictionary<Guid, List<TimelineRecordLogLine>>();
List<Guid> stepRecordIds = new List<Guid>(); // We need to keep lines in order List<Guid> stepRecordIds = new List<Guid>(); // We need to keep lines in order
int linesCounter = 0; int linesCounter = 0;
ConsoleLineInfo lineInfo; ConsoleLineInfo lineInfo;
@@ -222,7 +222,7 @@ namespace GitHub.Runner.Common
{ {
if (!stepsConsoleLines.ContainsKey(lineInfo.StepRecordId)) if (!stepsConsoleLines.ContainsKey(lineInfo.StepRecordId))
{ {
stepsConsoleLines[lineInfo.StepRecordId] = new List<string>(); stepsConsoleLines[lineInfo.StepRecordId] = new List<TimelineRecordLogLine>();
stepRecordIds.Add(lineInfo.StepRecordId); stepRecordIds.Add(lineInfo.StepRecordId);
} }
@@ -232,7 +232,7 @@ namespace GitHub.Runner.Common
lineInfo.Line = $"{lineInfo.Line.Substring(0, 1024)}..."; lineInfo.Line = $"{lineInfo.Line.Substring(0, 1024)}...";
} }
stepsConsoleLines[lineInfo.StepRecordId].Add(lineInfo.Line); stepsConsoleLines[lineInfo.StepRecordId].Add(new TimelineRecordLogLine(lineInfo.Line, lineInfo.LineNumber));
linesCounter++; linesCounter++;
// process at most about 500 lines of web console line during regular timer dequeue task. // process at most about 500 lines of web console line during regular timer dequeue task.
@@ -247,13 +247,13 @@ namespace GitHub.Runner.Common
{ {
// Split consolelines into batch, each batch will container at most 100 lines. // Split consolelines into batch, each batch will container at most 100 lines.
int batchCounter = 0; int batchCounter = 0;
List<List<string>> batchedLines = new List<List<string>>(); List<List<TimelineRecordLogLine>> batchedLines = new List<List<TimelineRecordLogLine>>();
foreach (var line in stepsConsoleLines[stepRecordId]) foreach (var line in stepsConsoleLines[stepRecordId])
{ {
var currentBatch = batchedLines.ElementAtOrDefault(batchCounter); var currentBatch = batchedLines.ElementAtOrDefault(batchCounter);
if (currentBatch == null) if (currentBatch == null)
{ {
batchedLines.Add(new List<string>()); batchedLines.Add(new List<TimelineRecordLogLine>());
currentBatch = batchedLines.ElementAt(batchCounter); currentBatch = batchedLines.ElementAt(batchCounter);
} }
@@ -275,7 +275,6 @@ namespace GitHub.Runner.Common
{ {
Trace.Info($"Skip {batchedLines.Count - 2} batches web console lines for last run"); Trace.Info($"Skip {batchedLines.Count - 2} batches web console lines for last run");
batchedLines = batchedLines.TakeLast(2).ToList(); batchedLines = batchedLines.TakeLast(2).ToList();
batchedLines[0].Insert(0, "...");
} }
int errorCount = 0; int errorCount = 0;
@@ -284,7 +283,15 @@ namespace GitHub.Runner.Common
try try
{ {
// we will not requeue failed batch, since the web console lines are time sensitive. // we will not requeue failed batch, since the web console lines are time sensitive.
await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch, default(CancellationToken)); if (batch[0].LineNumber.HasValue)
{
await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), batch[0].LineNumber.Value, default(CancellationToken));
}
else
{
await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), default(CancellationToken));
}
if (_firstConsoleOutputs) if (_firstConsoleOutputs)
{ {
HostContext.WritePerfCounter($"WorkerJobServerQueueAppendFirstConsoleOutput_{_planId.ToString()}"); HostContext.WritePerfCounter($"WorkerJobServerQueueAppendFirstConsoleOutput_{_planId.ToString()}");
@@ -653,13 +660,15 @@ namespace GitHub.Runner.Common
internal class ConsoleLineInfo internal class ConsoleLineInfo
{ {
public ConsoleLineInfo(Guid recordId, string line) public ConsoleLineInfo(Guid recordId, string line, long? lineNumber)
{ {
this.StepRecordId = recordId; this.StepRecordId = recordId;
this.Line = line; this.Line = line;
this.LineNumber = lineNumber;
} }
public Guid StepRecordId { get; set; } public Guid StepRecordId { get; set; }
public string Line { get; set; } public string Line { get; set; }
public long? LineNumber { get; set; }
} }
} }

View File

@@ -50,10 +50,6 @@ namespace GitHub.Runner.Common
// agent update // agent update
Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, int agentId, string currentState); Task<TaskAgent> UpdateAgentUpdateStateAsync(int agentPoolId, int agentId, string currentState);
// runner authorization url
Task<string> GetRunnerAuthUrlAsync(int runnerPoolId, int runnerId);
Task ReportRunnerAuthUrlErrorAsync(int runnerPoolId, int runnerId, string error);
} }
public sealed class RunnerServer : RunnerService, IRunnerServer public sealed class RunnerServer : RunnerService, IRunnerServer

View File

@@ -96,13 +96,14 @@ namespace GitHub.Runner.Common
Trace.Info($"WRITE: {message}"); Trace.Info($"WRITE: {message}");
if (!Silent) if (!Silent)
{ {
if(colorCode != null) if (colorCode != null)
{ {
Console.ForegroundColor = colorCode.Value; Console.ForegroundColor = colorCode.Value;
Console.Write(message); Console.Write(message);
Console.ResetColor(); Console.ResetColor();
} }
else { else
{
Console.Write(message); Console.Write(message);
} }
} }
@@ -120,13 +121,14 @@ namespace GitHub.Runner.Common
Trace.Info($"WRITE LINE: {line}"); Trace.Info($"WRITE LINE: {line}");
if (!Silent) if (!Silent)
{ {
if(colorCode != null) if (colorCode != null)
{ {
Console.ForegroundColor = colorCode.Value; Console.ForegroundColor = colorCode.Value;
Console.WriteLine(line); Console.WriteLine(line);
Console.ResetColor(); Console.ResetColor();
} }
else { else
{
Console.WriteLine(line); Console.WriteLine(line);
} }
} }

View File

@@ -0,0 +1,51 @@
using System;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Runner.Sdk;
using GitHub.Runner.Common;
namespace GitHub.Runner.Common.Util
{
public static class EncodingUtil
{
public static async Task SetEncoding(IHostContext hostContext, Tracing trace, CancellationToken cancellationToken)
{
#if OS_WINDOWS
try
{
if (Console.InputEncoding.CodePage != 65001)
{
using (var p = hostContext.CreateService<IProcessInvoker>())
{
// Use UTF8 code page
int exitCode = await p.ExecuteAsync(workingDirectory: hostContext.GetDirectory(WellKnownDirectory.Work),
fileName: WhichUtil.Which("chcp", true, trace),
arguments: "65001",
environment: null,
requireExitCodeZero: false,
outputEncoding: null,
killProcessOnCancel: false,
redirectStandardIn: null,
inheritConsoleHandler: true,
cancellationToken: cancellationToken);
if (exitCode == 0)
{
trace.Info("Successfully returned to code page 65001 (UTF8)");
}
else
{
trace.Warning($"'chcp 65001' failed with exit code {exitCode}");
}
}
}
}
catch (Exception ex)
{
trace.Warning($"'chcp 65001' failed with exception {ex.Message}");
}
#endif
// Dummy variable to prevent compiler error CS1998: "This async method lacks 'await' operators and will run synchronously..."
await Task.CompletedTask;
}
}
}

View File

@@ -0,0 +1,90 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Listener.Check
{
public sealed class ActionsCheck : RunnerService, ICheckExtension
{
private string _logFile = null;
public int Order => 2;
public string CheckName => "GitHub Actions Connection";
public string CheckDescription => "Make sure the actions runner have access to the GitHub Actions Service.";
public string CheckLog => _logFile;
public string HelpLink => "https://github.com/actions/runner/blob/main/docs/checks/actions.md";
public Type ExtensionType => typeof(ICheckExtension);
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_logFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Diag), StringUtil.Format("{0}_{1:yyyyMMdd-HHmmss}-utc.log", nameof(ActionsCheck), DateTime.UtcNow));
}
// runner access to actions service
public async Task<bool> RunCheck(string url, string pat)
{
await File.AppendAllLinesAsync(_logFile, HostContext.WarnLog());
await File.AppendAllLinesAsync(_logFile, HostContext.CheckProxy());
var checkTasks = new List<Task<CheckResult>>();
string githubApiUrl = null;
string actionsTokenServiceUrl = null;
string actionsPipelinesServiceUrl = null;
var urlBuilder = new UriBuilder(url);
if (UrlUtil.IsHostedServer(urlBuilder))
{
urlBuilder.Host = $"api.{urlBuilder.Host}";
urlBuilder.Path = "";
githubApiUrl = urlBuilder.Uri.AbsoluteUri;
actionsTokenServiceUrl = "https://vstoken.actions.githubusercontent.com/_apis/health";
actionsPipelinesServiceUrl = "https://pipelines.actions.githubusercontent.com/_apis/health";
}
else
{
urlBuilder.Path = "api/v3";
githubApiUrl = urlBuilder.Uri.AbsoluteUri;
urlBuilder.Path = "_services/vstoken/_apis/health";
actionsTokenServiceUrl = urlBuilder.Uri.AbsoluteUri;
urlBuilder.Path = "_services/pipelines/_apis/health";
actionsPipelinesServiceUrl = urlBuilder.Uri.AbsoluteUri;
}
// check github api
checkTasks.Add(CheckUtil.CheckDns(githubApiUrl));
checkTasks.Add(CheckUtil.CheckPing(githubApiUrl));
checkTasks.Add(HostContext.CheckHttpsRequests(githubApiUrl, pat, expectedHeader: "X-GitHub-Request-Id"));
// check actions token service
checkTasks.Add(CheckUtil.CheckDns(actionsTokenServiceUrl));
checkTasks.Add(CheckUtil.CheckPing(actionsTokenServiceUrl));
checkTasks.Add(HostContext.CheckHttpsRequests(actionsTokenServiceUrl, pat, expectedHeader: "x-vss-e2eid"));
// check actions pipelines service
checkTasks.Add(CheckUtil.CheckDns(actionsPipelinesServiceUrl));
checkTasks.Add(CheckUtil.CheckPing(actionsPipelinesServiceUrl));
checkTasks.Add(HostContext.CheckHttpsRequests(actionsPipelinesServiceUrl, pat, expectedHeader: "x-vss-e2eid"));
var result = true;
while (checkTasks.Count > 0)
{
var finishedCheckTask = await Task.WhenAny<CheckResult>(checkTasks);
var finishedCheck = await finishedCheckTask;
result = result && finishedCheck.Pass;
await File.AppendAllLinesAsync(_logFile, finishedCheck.Logs);
checkTasks.Remove(finishedCheckTask);
}
await Task.WhenAll(checkTasks);
return result;
}
}
}

View File

@@ -0,0 +1,351 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Tracing;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Net.NetworkInformation;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
using GitHub.Services.Common;
namespace GitHub.Runner.Listener.Check
{
public static class CheckUtil
{
public static List<string> WarnLog(this IHostContext hostContext)
{
var logs = new List<string>();
logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
logs.Add($"{DateTime.UtcNow.ToString("O")} **** !!! WARNING !!! ");
logs.Add($"{DateTime.UtcNow.ToString("O")} **** DO NOT share the log in public place! The log may contains secrets in plain text. ");
logs.Add($"{DateTime.UtcNow.ToString("O")} **** !!! WARNING !!! ");
logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
return logs;
}
public static List<string> CheckProxy(this IHostContext hostContext)
{
var logs = new List<string>();
if (!string.IsNullOrEmpty(hostContext.WebProxy.HttpProxyAddress) ||
!string.IsNullOrEmpty(hostContext.WebProxy.HttpsProxyAddress))
{
logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
logs.Add($"{DateTime.UtcNow.ToString("O")} **** Runner is behind web proxy {hostContext.WebProxy.HttpsProxyAddress ?? hostContext.WebProxy.HttpProxyAddress} ");
logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
}
return logs;
}
public static async Task<CheckResult> CheckDns(string targetUrl)
{
var result = new CheckResult();
var url = new Uri(targetUrl);
try
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Try DNS lookup for {url.Host} ");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
IPHostEntry host = await Dns.GetHostEntryAsync(url.Host);
foreach (var address in host.AddressList)
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Resolved DNS for {url.Host} to '{address}'");
}
result.Pass = true;
}
catch (Exception ex)
{
result.Pass = false;
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Resolved DNS for {url.Host} failed with error: {ex}");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
}
return result;
}
public static async Task<CheckResult> CheckPing(string targetUrl)
{
var result = new CheckResult();
var url = new Uri(targetUrl);
try
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Try ping {url.Host} ");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
using (var ping = new Ping())
{
var reply = await ping.SendPingAsync(url.Host);
if (reply.Status == IPStatus.Success)
{
result.Pass = true;
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Ping {url.Host} ({reply.Address}) succeed within to '{reply.RoundtripTime} ms'");
}
else
{
result.Pass = false;
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Ping {url.Host} ({reply.Address}) failed with '{reply.Status}'");
}
}
}
catch (Exception ex)
{
result.Pass = false;
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Ping api.github.com failed with error: {ex}");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
}
return result;
}
public static async Task<CheckResult> CheckHttpsRequests(this IHostContext hostContext, string url, string pat, string expectedHeader)
{
var result = new CheckResult();
try
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Send HTTPS Request to {url} ");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
using (var _ = new HttpEventSourceListener(result.Logs))
using (var httpClientHandler = hostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
httpClient.DefaultRequestHeaders.UserAgent.AddRange(hostContext.UserAgents);
if (!string.IsNullOrEmpty(pat))
{
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("token", pat);
}
var response = await httpClient.GetAsync(url);
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Http status code: {response.StatusCode}");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Http response headers: {response.Headers}");
var responseContent = await response.Content.ReadAsStringAsync();
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Http response body: {responseContent}");
if (response.IsSuccessStatusCode)
{
if (response.Headers.Contains(expectedHeader))
{
result.Pass = true;
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Http request 'GET' to {url} succeed");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ");
}
else
{
result.Pass = false;
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Http request 'GET' to {url} succeed but doesn't have expected HTTP Header.");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ");
}
}
else
{
result.Pass = false;
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Http request 'GET' to {url} failed with {response.StatusCode}");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ");
}
}
}
catch (Exception ex)
{
result.Pass = false;
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Https request 'GET' to {url} failed with error: {ex}");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
}
return result;
}
public static async Task<CheckResult> DownloadExtraCA(this IHostContext hostContext, string url, string pat)
{
var result = new CheckResult();
try
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Download SSL Certificate from {url} ");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
var uri = new Uri(url);
var env = new Dictionary<string, string>()
{
{ "HOSTNAME", uri.Host },
{ "PORT", uri.IsDefaultPort ? (uri.Scheme.ToLowerInvariant() == "https" ? "443" : "80") : uri.Port.ToString() },
{ "PATH", uri.AbsolutePath },
{ "PAT", pat }
};
var proxy = hostContext.WebProxy.GetProxy(uri);
if (proxy != null)
{
env["PROXYHOST"] = proxy.Host;
env["PROXYPORT"] = proxy.IsDefaultPort ? (proxy.Scheme.ToLowerInvariant() == "https" ? "443" : "80") : proxy.Port.ToString();
if (hostContext.WebProxy.HttpProxyUsername != null ||
hostContext.WebProxy.HttpsProxyUsername != null)
{
env["PROXYUSERNAME"] = hostContext.WebProxy.HttpProxyUsername ?? hostContext.WebProxy.HttpsProxyUsername;
env["PROXYPASSWORD"] = hostContext.WebProxy.HttpProxyPassword ?? hostContext.WebProxy.HttpsProxyPassword;
}
else
{
env["PROXYUSERNAME"] = "";
env["PROXYPASSWORD"] = "";
}
}
else
{
env["PROXYHOST"] = "";
env["PROXYPORT"] = "";
env["PROXYUSERNAME"] = "";
env["PROXYPASSWORD"] = "";
}
using (var processInvoker = hostContext.CreateService<IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} [STDOUT] {args.Data}");
}
});
processInvoker.ErrorDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} [STDERR] {args.Data}");
}
});
var downloadCertScript = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Bin), "checkScripts", "downloadCert");
var node12 = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Externals), "node12", "bin", $"node{IOUtil.ExeExtension}");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Run '{node12} \"{downloadCertScript}\"' ");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} {StringUtil.ConvertToJson(env)}");
await processInvoker.ExecuteAsync(
hostContext.GetDirectory(WellKnownDirectory.Root),
node12,
$"\"{downloadCertScript}\"",
env,
true,
CancellationToken.None);
}
result.Pass = true;
}
catch (Exception ex)
{
result.Pass = false;
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Download SSL Certificate from '{url}' failed with error: {ex}");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
}
return result;
}
}
// EventSource listener for dotnet debug trace for HTTP and SSL
public sealed class HttpEventSourceListener : EventListener
{
private readonly List<string> _logs;
private readonly object _lock = new object();
private readonly Dictionary<string, HashSet<string>> _ignoredEvent = new Dictionary<string, HashSet<string>>
{
{
"Private.InternalDiagnostics.System.Net.Http",
new HashSet<string>
{
"Info",
"Associate"
}
},
{
"Private.InternalDiagnostics.System.Net.Security",
new HashSet<string>
{
"Info",
"SslStreamCtor",
"SecureChannelCtor",
"NoDelegateNoClientCert",
"CertsAfterFiltering",
"UsingCachedCredential",
"SspiSelectedCipherSuite"
}
}
};
public HttpEventSourceListener(List<string> logs)
{
_logs = logs;
if (Environment.GetEnvironmentVariable("ACTIONS_RUNNER_TRACE_ALL_HTTP_EVENT") == "1")
{
_ignoredEvent.Clear();
}
}
protected override void OnEventSourceCreated(EventSource eventSource)
{
base.OnEventSourceCreated(eventSource);
if (eventSource.Name == "Private.InternalDiagnostics.System.Net.Http" ||
eventSource.Name == "Private.InternalDiagnostics.System.Net.Security")
{
EnableEvents(eventSource, EventLevel.Verbose, EventKeywords.All);
}
}
protected override void OnEventWritten(EventWrittenEventArgs eventData)
{
base.OnEventWritten(eventData);
lock (_lock)
{
if (_ignoredEvent.TryGetValue(eventData.EventSource.Name, out var ignored) &&
ignored.Contains(eventData.EventName))
{
return;
}
_logs.Add($"{DateTime.UtcNow.ToString("O")} [START {eventData.EventSource.Name} - {eventData.EventName}]");
_logs.AddRange(eventData.Payload.Select(x => string.Join(Environment.NewLine, x.ToString().Split(Environment.NewLine).Select(y => $"{DateTime.UtcNow.ToString("O")} {y}"))));
_logs.Add($"{DateTime.UtcNow.ToString("O")} [END {eventData.EventSource.Name} - {eventData.EventName}]");
}
}
}
}

View File

@@ -0,0 +1,171 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Listener.Check
{
public sealed class GitCheck : RunnerService, ICheckExtension
{
private string _logFile = null;
private string _gitPath = null;
public int Order => 3;
public string CheckName => "Git Certificate/Proxy Validation";
public string CheckDescription => "Make sure the git cli can access to GitHub.com or the GitHub Enterprise Server.";
public string CheckLog => _logFile;
public string HelpLink => "https://github.com/actions/runner/blob/main/docs/checks/git.md";
public Type ExtensionType => typeof(ICheckExtension);
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_logFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Diag), StringUtil.Format("{0}_{1:yyyyMMdd-HHmmss}-utc.log", nameof(GitCheck), DateTime.UtcNow));
_gitPath = WhichUtil.Which("git");
}
// git access to ghes/gh
public async Task<bool> RunCheck(string url, string pat)
{
await File.AppendAllLinesAsync(_logFile, HostContext.WarnLog());
await File.AppendAllLinesAsync(_logFile, HostContext.CheckProxy());
if (string.IsNullOrEmpty(_gitPath))
{
await File.AppendAllLinesAsync(_logFile, new[] { $"{DateTime.UtcNow.ToString("O")} Can't verify git with GitHub.com or GitHub Enterprise Server since git is not installed." });
return false;
}
var checkGit = await CheckGit(url, pat);
var result = checkGit.Pass;
await File.AppendAllLinesAsync(_logFile, checkGit.Logs);
// try fix SSL error by providing extra CA certificate.
if (checkGit.SslError)
{
await File.AppendAllLinesAsync(_logFile, new[] { $"{DateTime.UtcNow.ToString("O")} Try fix SSL error by providing extra CA certificate." });
var downloadCert = await HostContext.DownloadExtraCA(url, pat);
await File.AppendAllLinesAsync(_logFile, downloadCert.Logs);
if (downloadCert.Pass)
{
var recheckGit = await CheckGit(url, pat, extraCA: true);
await File.AppendAllLinesAsync(_logFile, recheckGit.Logs);
if (recheckGit.Pass)
{
await File.AppendAllLinesAsync(_logFile, new[] { $"{DateTime.UtcNow.ToString("O")} Fixed SSL error by providing extra CA certs." });
}
}
}
return result;
}
private async Task<CheckResult> CheckGit(string url, string pat, bool extraCA = false)
{
var result = new CheckResult();
try
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Validate server cert and proxy configuration with Git ");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
var repoUrlBuilder = new UriBuilder(url);
repoUrlBuilder.Path = "actions/checkout";
repoUrlBuilder.UserName = "gh";
repoUrlBuilder.Password = pat;
var gitProxy = "";
var proxy = HostContext.WebProxy.GetProxy(repoUrlBuilder.Uri);
if (proxy != null)
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Runner is behind http proxy '{proxy.AbsoluteUri}'");
if (HostContext.WebProxy.HttpProxyUsername != null ||
HostContext.WebProxy.HttpsProxyUsername != null)
{
var proxyUrlWithCred = UrlUtil.GetCredentialEmbeddedUrl(
proxy,
HostContext.WebProxy.HttpProxyUsername ?? HostContext.WebProxy.HttpsProxyUsername,
HostContext.WebProxy.HttpProxyPassword ?? HostContext.WebProxy.HttpsProxyPassword);
gitProxy = $"-c http.proxy={proxyUrlWithCred}";
}
else
{
gitProxy = $"-c http.proxy={proxy.AbsoluteUri}";
}
}
using (var processInvoker = HostContext.CreateService<IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} {args.Data}");
}
});
processInvoker.ErrorDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} {args.Data}");
}
});
var gitArgs = $"{gitProxy} ls-remote --exit-code {repoUrlBuilder.Uri.AbsoluteUri} HEAD";
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Run 'git {gitArgs}' ");
var env = new Dictionary<string, string>
{
{ "GIT_TRACE", "1" },
{ "GIT_CURL_VERBOSE", "1" }
};
if (extraCA)
{
env["GIT_SSL_CAINFO"] = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), "download_ca_cert.pem");
}
await processInvoker.ExecuteAsync(
HostContext.GetDirectory(WellKnownDirectory.Root),
_gitPath,
gitArgs,
env,
true,
CancellationToken.None);
}
result.Pass = true;
}
catch (Exception ex)
{
result.Pass = false;
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** git ls-remote failed with error: {ex}");
if (result.Logs.Any(x => x.Contains("SSL Certificate problem", StringComparison.OrdinalIgnoreCase)))
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** git ls-remote failed due to SSL cert issue.");
result.SslError = true;
}
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
}
return result;
}
}
}

View File

@@ -0,0 +1,30 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using GitHub.Runner.Common;
namespace GitHub.Runner.Listener.Check
{
public interface ICheckExtension : IExtension
{
int Order { get; }
string CheckName { get; }
string CheckDescription { get; }
string CheckLog { get; }
string HelpLink { get; }
Task<bool> RunCheck(string url, string pat);
}
public class CheckResult
{
public CheckResult()
{
Logs = new List<string>();
}
public bool Pass { get; set; }
public bool SslError { get; set; }
public List<string> Logs { get; set; }
}
}

View File

@@ -0,0 +1,59 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Listener.Check
{
public sealed class InternetCheck : RunnerService, ICheckExtension
{
private string _logFile = null;
public int Order => 1;
public string CheckName => "Internet Connection";
public string CheckDescription => "Make sure the actions runner have access to public internet.";
public string CheckLog => _logFile;
public string HelpLink => "https://github.com/actions/runner/blob/main/docs/checks/internet.md";
public Type ExtensionType => typeof(ICheckExtension);
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_logFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Diag), StringUtil.Format("{0}_{1:yyyyMMdd-HHmmss}-utc.log", nameof(InternetCheck), DateTime.UtcNow));
}
// check runner access to api.github.com
public async Task<bool> RunCheck(string url, string pat)
{
await File.AppendAllLinesAsync(_logFile, HostContext.WarnLog());
await File.AppendAllLinesAsync(_logFile, HostContext.CheckProxy());
var checkTasks = new List<Task<CheckResult>>();
checkTasks.Add(CheckUtil.CheckDns("https://api.github.com"));
checkTasks.Add(CheckUtil.CheckPing("https://api.github.com"));
// We don't need to pass a PAT since it might be a token for GHES.
checkTasks.Add(HostContext.CheckHttpsRequests("https://api.github.com", pat: null, expectedHeader: "X-GitHub-Request-Id"));
var result = true;
while (checkTasks.Count > 0)
{
var finishedCheckTask = await Task.WhenAny<CheckResult>(checkTasks);
var finishedCheck = await finishedCheckTask;
result = result && finishedCheck.Pass;
await File.AppendAllLinesAsync(_logFile, finishedCheck.Logs);
checkTasks.Remove(finishedCheckTask);
}
await Task.WhenAll(checkTasks);
return result;
}
}
}

View File

@@ -0,0 +1,181 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
namespace GitHub.Runner.Listener.Check
{
public sealed class NodeJsCheck : RunnerService, ICheckExtension
{
private string _logFile = null;
public int Order => 4;
public string CheckName => "Node.js Certificate/Proxy Validation";
public string CheckDescription => "Make sure the node.js have access to GitHub.com or the GitHub Enterprise Server.";
public string CheckLog => _logFile;
public string HelpLink => "https://github.com/actions/runner/blob/main/docs/checks/nodejs.md";
public Type ExtensionType => typeof(ICheckExtension);
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_logFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Diag), StringUtil.Format("{0}_{1:yyyyMMdd-HHmmss}-utc.log", nameof(NodeJsCheck), DateTime.UtcNow));
}
// node access to ghes/gh
public async Task<bool> RunCheck(string url, string pat)
{
await File.AppendAllLinesAsync(_logFile, HostContext.WarnLog());
await File.AppendAllLinesAsync(_logFile, HostContext.CheckProxy());
// Request to github.com or ghes server
var urlBuilder = new UriBuilder(url);
if (UrlUtil.IsHostedServer(urlBuilder))
{
urlBuilder.Host = $"api.{urlBuilder.Host}";
urlBuilder.Path = "";
}
else
{
urlBuilder.Path = "api/v3";
}
var checkNode = await CheckNodeJs(urlBuilder.Uri.AbsoluteUri, pat);
var result = checkNode.Pass;
await File.AppendAllLinesAsync(_logFile, checkNode.Logs);
// try fix SSL error by providing extra CA certificate.
if (checkNode.SslError)
{
var downloadCert = await HostContext.DownloadExtraCA(urlBuilder.Uri.AbsoluteUri, pat);
await File.AppendAllLinesAsync(_logFile, downloadCert.Logs);
if (downloadCert.Pass)
{
var recheckNode = await CheckNodeJs(urlBuilder.Uri.AbsoluteUri, pat, extraCA: true);
await File.AppendAllLinesAsync(_logFile, recheckNode.Logs);
if (recheckNode.Pass)
{
await File.AppendAllLinesAsync(_logFile, new[] { $"{DateTime.UtcNow.ToString("O")} Fixed SSL error by providing extra CA certs." });
}
}
}
return result;
}
private async Task<CheckResult> CheckNodeJs(string url, string pat, bool extraCA = false)
{
var result = new CheckResult();
try
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Make Http request to {url} using node.js ");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
// Request to github.com or ghes server
Uri requestUrl = new Uri(url);
var env = new Dictionary<string, string>()
{
{ "HOSTNAME", requestUrl.Host },
{ "PORT", requestUrl.IsDefaultPort ? (requestUrl.Scheme.ToLowerInvariant() == "https" ? "443" : "80") : requestUrl.Port.ToString() },
{ "PATH", requestUrl.AbsolutePath },
{ "PAT", pat }
};
var proxy = HostContext.WebProxy.GetProxy(requestUrl);
if (proxy != null)
{
env["PROXYHOST"] = proxy.Host;
env["PROXYPORT"] = proxy.IsDefaultPort ? (proxy.Scheme.ToLowerInvariant() == "https" ? "443" : "80") : proxy.Port.ToString();
if (HostContext.WebProxy.HttpProxyUsername != null ||
HostContext.WebProxy.HttpsProxyUsername != null)
{
env["PROXYUSERNAME"] = HostContext.WebProxy.HttpProxyUsername ?? HostContext.WebProxy.HttpsProxyUsername;
env["PROXYPASSWORD"] = HostContext.WebProxy.HttpProxyPassword ?? HostContext.WebProxy.HttpsProxyPassword;
}
else
{
env["PROXYUSERNAME"] = "";
env["PROXYPASSWORD"] = "";
}
}
else
{
env["PROXYHOST"] = "";
env["PROXYPORT"] = "";
env["PROXYUSERNAME"] = "";
env["PROXYPASSWORD"] = "";
}
if (extraCA)
{
env["NODE_EXTRA_CA_CERTS"] = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), "download_ca_cert.pem");
}
using (var processInvoker = HostContext.CreateService<IProcessInvoker>())
{
processInvoker.OutputDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} [STDOUT] {args.Data}");
}
});
processInvoker.ErrorDataReceived += new EventHandler<ProcessDataReceivedEventArgs>((sender, args) =>
{
if (!string.IsNullOrEmpty(args.Data))
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} [STDERR] {args.Data}");
}
});
var makeWebRequestScript = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Bin), "checkScripts", "makeWebRequest.js");
var node12 = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Externals), "node12", "bin", $"node{IOUtil.ExeExtension}");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} Run '{node12} \"{makeWebRequestScript}\"' ");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} {StringUtil.ConvertToJson(env)}");
await processInvoker.ExecuteAsync(
HostContext.GetDirectory(WellKnownDirectory.Root),
node12,
$"\"{makeWebRequestScript}\"",
env,
true,
CancellationToken.None);
}
result.Pass = true;
}
catch (Exception ex)
{
result.Pass = false;
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Make https request to {url} using node.js failed with error: {ex}");
if (result.Logs.Any(x => x.Contains("UNABLE_TO_VERIFY_LEAF_SIGNATURE") ||
x.Contains("UNABLE_TO_GET_ISSUER_CERT_LOCALLY") ||
x.Contains("SELF_SIGNED_CERT_IN_CHAIN")))
{
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** Https request failed due to SSL cert issue.");
result.SslError = true;
}
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} **** ****");
result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************");
}
return result;
}
}
}

View File

@@ -27,6 +27,7 @@ namespace GitHub.Runner.Listener
private readonly string[] validFlags = private readonly string[] validFlags =
{ {
Constants.Runner.CommandLine.Flags.Check,
Constants.Runner.CommandLine.Flags.Commit, Constants.Runner.CommandLine.Flags.Commit,
Constants.Runner.CommandLine.Flags.Help, Constants.Runner.CommandLine.Flags.Help,
Constants.Runner.CommandLine.Flags.Replace, Constants.Runner.CommandLine.Flags.Replace,
@@ -42,7 +43,8 @@ namespace GitHub.Runner.Listener
Constants.Runner.CommandLine.Args.Labels, Constants.Runner.CommandLine.Args.Labels,
Constants.Runner.CommandLine.Args.MonitorSocketAddress, Constants.Runner.CommandLine.Args.MonitorSocketAddress,
Constants.Runner.CommandLine.Args.Name, Constants.Runner.CommandLine.Args.Name,
Constants.Runner.CommandLine.Args.Pool, Constants.Runner.CommandLine.Args.PAT,
Constants.Runner.CommandLine.Args.RunnerGroup,
Constants.Runner.CommandLine.Args.StartupType, Constants.Runner.CommandLine.Args.StartupType,
Constants.Runner.CommandLine.Args.Token, Constants.Runner.CommandLine.Args.Token,
Constants.Runner.CommandLine.Args.Url, Constants.Runner.CommandLine.Args.Url,
@@ -59,6 +61,7 @@ namespace GitHub.Runner.Listener
public bool Warmup => TestCommand(Constants.Runner.CommandLine.Commands.Warmup); public bool Warmup => TestCommand(Constants.Runner.CommandLine.Commands.Warmup);
// Flags. // Flags.
public bool Check => TestFlag(Constants.Runner.CommandLine.Flags.Check);
public bool Commit => TestFlag(Constants.Runner.CommandLine.Flags.Commit); public bool Commit => TestFlag(Constants.Runner.CommandLine.Flags.Commit);
public bool Help => TestFlag(Constants.Runner.CommandLine.Flags.Help); public bool Help => TestFlag(Constants.Runner.CommandLine.Flags.Help);
public bool Unattended => TestFlag(Constants.Runner.CommandLine.Flags.Unattended); public bool Unattended => TestFlag(Constants.Runner.CommandLine.Flags.Unattended);
@@ -169,6 +172,15 @@ namespace GitHub.Runner.Listener
validator: Validators.NonEmptyValidator); validator: Validators.NonEmptyValidator);
} }
public string GetRunnerGroupName(string defaultPoolName = null)
{
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.RunnerGroup,
description: "Enter the name of the runner group to add this runner to:",
defaultValue: defaultPoolName ?? "default",
validator: Validators.NonEmptyValidator);
}
public string GetToken() public string GetToken()
{ {
return GetArgOrPrompt( return GetArgOrPrompt(
@@ -178,6 +190,22 @@ namespace GitHub.Runner.Listener
validator: Validators.NonEmptyValidator); validator: Validators.NonEmptyValidator);
} }
public string GetGitHubPersonalAccessToken(bool required = false)
{
if (required)
{
return GetArgOrPrompt(
name: Constants.Runner.CommandLine.Args.PAT,
description: "What is your GitHub personal access token?",
defaultValue: string.Empty,
validator: Validators.NonEmptyValidator);
}
else
{
return GetArg(name: Constants.Runner.CommandLine.Args.PAT);
}
}
public string GetRunnerRegisterToken() public string GetRunnerRegisterToken()
{ {
return GetArgOrPrompt( return GetArgOrPrompt(

View File

@@ -4,7 +4,6 @@ using GitHub.Runner.Common.Util;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;
using GitHub.Services.Common; using GitHub.Services.Common;
using GitHub.Services.OAuth; using GitHub.Services.OAuth;
using GitHub.Services.WebApi;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
@@ -12,6 +11,7 @@ using System.Net.Http;
using System.Net.Http.Headers; using System.Net.Http.Headers;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text;
using System.Threading.Tasks; using System.Threading.Tasks;
namespace GitHub.Runner.Listener.Configuration namespace GitHub.Runner.Listener.Configuration
@@ -92,9 +92,11 @@ namespace GitHub.Runner.Listener.Configuration
_term.WriteSection("Authentication"); _term.WriteSection("Authentication");
while (true) while (true)
{ {
// Get the URL // When testing against a dev deployment of Actions Service, set this environment variable
var useDevActionsServiceUrl = Environment.GetEnvironmentVariable("USE_DEV_ACTIONS_SERVICE_URL");
var inputUrl = command.GetUrl(); var inputUrl = command.GetUrl();
if (inputUrl.Contains("codedev.ms", StringComparison.OrdinalIgnoreCase)) if (inputUrl.Contains("codedev.ms", StringComparison.OrdinalIgnoreCase)
|| useDevActionsServiceUrl != null)
{ {
runnerSettings.ServerUrl = inputUrl; runnerSettings.ServerUrl = inputUrl;
// Get the credentials // Get the credentials
@@ -105,8 +107,8 @@ namespace GitHub.Runner.Listener.Configuration
else else
{ {
runnerSettings.GitHubUrl = inputUrl; runnerSettings.GitHubUrl = inputUrl;
var githubToken = command.GetRunnerRegisterToken(); var registerToken = await GetRunnerTokenAsync(command, inputUrl, "registration");
GitHubAuthResult authResult = await GetTenantCredential(inputUrl, githubToken, Constants.RunnerEvent.Register); GitHubAuthResult authResult = await GetTenantCredential(inputUrl, registerToken, Constants.RunnerEvent.Register);
runnerSettings.ServerUrl = authResult.TenantUrl; runnerSettings.ServerUrl = authResult.TenantUrl;
creds = authResult.ToVssCredentials(); creds = authResult.ToVssCredentials();
Trace.Info("cred retrieved via GitHub auth"); Trace.Info("cred retrieved via GitHub auth");
@@ -115,7 +117,20 @@ namespace GitHub.Runner.Listener.Configuration
try try
{ {
// Determine the service deployment type based on connection data. (Hosted/OnPremises) // Determine the service deployment type based on connection data. (Hosted/OnPremises)
runnerSettings.IsHostedServer = runnerSettings.GitHubUrl == null || IsHostedServer(new UriBuilder(runnerSettings.GitHubUrl)); runnerSettings.IsHostedServer = runnerSettings.GitHubUrl == null || UrlUtil.IsHostedServer(new UriBuilder(runnerSettings.GitHubUrl));
// Warn if the Actions server url and GHES server url has different Host
if (!runnerSettings.IsHostedServer)
{
// Example actionsServerUrl is https://my-ghes/_services/pipelines/[...]
// Example githubServerUrl is https://my-ghes
var actionsServerUrl = new Uri(runnerSettings.ServerUrl);
var githubServerUrl = new Uri(runnerSettings.GitHubUrl);
if (!string.Equals(actionsServerUrl.Authority, githubServerUrl.Authority, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException($"GitHub Actions is not properly configured in GHES. GHES url: {runnerSettings.GitHubUrl}, Actions url: {runnerSettings.ServerUrl}.");
}
}
// Validate can connect. // Validate can connect.
await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), creds); await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), creds);
@@ -144,17 +159,34 @@ namespace GitHub.Runner.Listener.Configuration
_term.WriteSection("Runner Registration"); _term.WriteSection("Runner Registration");
//Get all the agent pools, and select the first private pool // If we have more than one runner group available, allow the user to specify which one to be added into
string poolName = null;
TaskAgentPool agentPool = null;
List<TaskAgentPool> agentPools = await _runnerServer.GetAgentPoolsAsync(); List<TaskAgentPool> agentPools = await _runnerServer.GetAgentPoolsAsync();
TaskAgentPool agentPool = agentPools?.Where(x => x.IsHosted == false).FirstOrDefault(); TaskAgentPool defaultPool = agentPools?.Where(x => x.IsInternal).FirstOrDefault();
if (agentPool == null) if (agentPools?.Where(x => !x.IsHosted).Count() > 1)
{ {
throw new TaskAgentPoolNotFoundException($"Could not find any private pool. Contact support."); poolName = command.GetRunnerGroupName(defaultPool?.Name);
_term.WriteLine();
agentPool = agentPools.Where(x => string.Equals(poolName, x.Name, StringComparison.OrdinalIgnoreCase) && !x.IsHosted).FirstOrDefault();
} }
else else
{ {
Trace.Info("Found a private pool with id {1} and name {2}", agentPool.Id, agentPool.Name); agentPool = defaultPool;
}
if (agentPool == null && poolName == null)
{
throw new TaskAgentPoolNotFoundException($"Could not find any self-hosted runner groups. Contact support.");
}
else if (agentPool == null && poolName != null)
{
throw new TaskAgentPoolNotFoundException($"Could not find any self-hosted runner group named \"{poolName}\".");
}
else
{
Trace.Info("Found a self-hosted runner group with id {1} and name {2}", agentPool.Id, agentPool.Name);
runnerSettings.PoolId = agentPool.Id; runnerSettings.PoolId = agentPool.Id;
runnerSettings.PoolName = agentPool.Name; runnerSettings.PoolName = agentPool.Name;
} }
@@ -195,7 +227,7 @@ namespace GitHub.Runner.Listener.Configuration
else if (command.Unattended) else if (command.Unattended)
{ {
// if not replace and it is unattended config. // if not replace and it is unattended config.
throw new TaskAgentExistsException($"Pool {runnerSettings.PoolId} already contains a runner with name {runnerSettings.AgentName}."); throw new TaskAgentExistsException($"A runner exists with the same name {runnerSettings.AgentName}.");
} }
} }
else else
@@ -219,36 +251,11 @@ namespace GitHub.Runner.Listener.Configuration
// Add Agent Id to settings // Add Agent Id to settings
runnerSettings.AgentId = agent.Id; runnerSettings.AgentId = agent.Id;
// respect the serverUrl resolve by server.
// in case of agent configured using collection url instead of account url.
string agentServerUrl;
if (agent.Properties.TryGetValidatedValue<string>("ServerUrl", out agentServerUrl) &&
!string.IsNullOrEmpty(agentServerUrl))
{
Trace.Info($"Agent server url resolve by server: '{agentServerUrl}'.");
// we need make sure the Schema/Host/Port component of the url remain the same.
UriBuilder inputServerUrl = new UriBuilder(runnerSettings.ServerUrl);
UriBuilder serverReturnedServerUrl = new UriBuilder(agentServerUrl);
if (Uri.Compare(inputServerUrl.Uri, serverReturnedServerUrl.Uri, UriComponents.SchemeAndServer, UriFormat.Unescaped, StringComparison.OrdinalIgnoreCase) != 0)
{
inputServerUrl.Path = serverReturnedServerUrl.Path;
Trace.Info($"Replace server returned url's scheme://host:port component with user input server url's scheme://host:port: '{inputServerUrl.Uri.AbsoluteUri}'.");
runnerSettings.ServerUrl = inputServerUrl.Uri.AbsoluteUri;
}
else
{
runnerSettings.ServerUrl = agentServerUrl;
}
}
// See if the server supports our OAuth key exchange for credentials // See if the server supports our OAuth key exchange for credentials
if (agent.Authorization != null && if (agent.Authorization != null &&
agent.Authorization.ClientId != Guid.Empty && agent.Authorization.ClientId != Guid.Empty &&
agent.Authorization.AuthorizationUrl != null) agent.Authorization.AuthorizationUrl != null)
{ {
UriBuilder configServerUrl = new UriBuilder(runnerSettings.ServerUrl);
UriBuilder oauthEndpointUrlBuilder = new UriBuilder(agent.Authorization.AuthorizationUrl);
var credentialData = new CredentialData var credentialData = new CredentialData
{ {
Scheme = Constants.Configuration.OAuth, Scheme = Constants.Configuration.OAuth,
@@ -256,7 +263,7 @@ namespace GitHub.Runner.Listener.Configuration
{ {
{ "clientId", agent.Authorization.ClientId.ToString("D") }, { "clientId", agent.Authorization.ClientId.ToString("D") },
{ "authorizationUrl", agent.Authorization.AuthorizationUrl.AbsoluteUri }, { "authorizationUrl", agent.Authorization.AuthorizationUrl.AbsoluteUri },
{ "oauthEndpointUrl", oauthEndpointUrlBuilder.Uri.AbsoluteUri }, { "requireFipsCryptography", agent.Properties.GetValue("RequireFipsCryptography", false).ToString() }
}, },
}; };
@@ -367,8 +374,8 @@ namespace GitHub.Runner.Listener.Configuration
} }
else else
{ {
var githubToken = command.GetRunnerDeletionToken(); var deletionToken = await GetRunnerTokenAsync(command, settings.GitHubUrl, "remove");
GitHubAuthResult authResult = await GetTenantCredential(settings.GitHubUrl, githubToken, Constants.RunnerEvent.Remove); GitHubAuthResult authResult = await GetTenantCredential(settings.GitHubUrl, deletionToken, Constants.RunnerEvent.Remove);
creds = authResult.ToVssCredentials(); creds = authResult.ToVssCredentials();
Trace.Info("cred retrieved via GitHub auth"); Trace.Info("cred retrieved via GitHub auth");
} }
@@ -462,7 +469,7 @@ namespace GitHub.Runner.Listener.Configuration
// update should replace the existing labels // update should replace the existing labels
agent.Version = BuildConstants.RunnerPackage.Version; agent.Version = BuildConstants.RunnerPackage.Version;
agent.OSDescription = RuntimeInformation.OSDescription; agent.OSDescription = RuntimeInformation.OSDescription;
agent.Labels.Clear(); agent.Labels.Clear();
agent.Labels.Add(new AgentLabel("self-hosted", LabelType.System)); agent.Labels.Add(new AgentLabel("self-hosted", LabelType.System));
@@ -473,7 +480,7 @@ namespace GitHub.Runner.Listener.Configuration
{ {
agent.Labels.Add(new AgentLabel(userLabel, LabelType.User)); agent.Labels.Add(new AgentLabel(userLabel, LabelType.User));
} }
return agent; return agent;
} }
@@ -502,18 +509,107 @@ namespace GitHub.Runner.Listener.Configuration
return agent; return agent;
} }
private bool IsHostedServer(UriBuilder gitHubUrl) private async Task<string> GetRunnerTokenAsync(CommandSettings command, string githubUrl, string tokenType)
{ {
return string.Equals(gitHubUrl.Host, "github.com", StringComparison.OrdinalIgnoreCase) || var githubPAT = command.GetGitHubPersonalAccessToken();
string.Equals(gitHubUrl.Host, "www.github.com", StringComparison.OrdinalIgnoreCase) || var runnerToken = string.Empty;
string.Equals(gitHubUrl.Host, "github.localhost", StringComparison.OrdinalIgnoreCase); if (!string.IsNullOrEmpty(githubPAT))
{
Trace.Info($"Retriving runner {tokenType} token using GitHub PAT.");
var jitToken = await GetJITRunnerTokenAsync(githubUrl, githubPAT, tokenType);
Trace.Info($"Retrived runner {tokenType} token is good to {jitToken.ExpiresAt}.");
HostContext.SecretMasker.AddValue(jitToken.Token);
runnerToken = jitToken.Token;
}
if (string.IsNullOrEmpty(runnerToken))
{
if (string.Equals("registration", tokenType, StringComparison.OrdinalIgnoreCase))
{
runnerToken = command.GetRunnerRegisterToken();
}
else
{
runnerToken = command.GetRunnerDeletionToken();
}
}
return runnerToken;
}
private async Task<GitHubRunnerRegisterToken> GetJITRunnerTokenAsync(string githubUrl, string githubToken, string tokenType)
{
var githubApiUrl = "";
var gitHubUrlBuilder = new UriBuilder(githubUrl);
var path = gitHubUrlBuilder.Path.Split('/', '\\', StringSplitOptions.RemoveEmptyEntries);
if (path.Length == 1)
{
// org runner
if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
{
githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/orgs/{path[0]}/actions/runners/{tokenType}-token";
}
else
{
githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/orgs/{path[0]}/actions/runners/{tokenType}-token";
}
}
else if (path.Length == 2)
{
// repo or enterprise runner.
var repoScope = "repos/";
if (string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase))
{
repoScope = "";
}
if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
{
githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{repoScope}{path[0]}/{path[1]}/actions/runners/{tokenType}-token";
}
else
{
githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{repoScope}{path[0]}/{path[1]}/actions/runners/{tokenType}-token";
}
}
else
{
throw new ArgumentException($"'{githubUrl}' should point to an org or repository.");
}
using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler))
{
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"github:{githubToken}"));
HostContext.SecretMasker.AddValue(base64EncodingToken);
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("basic", base64EncodingToken);
httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
httpClient.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.github.v3+json");
var response = await httpClient.PostAsync(githubApiUrl, new StringContent(string.Empty));
if (response.IsSuccessStatusCode)
{
Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var jsonResponse = await response.Content.ReadAsStringAsync();
return StringUtil.ConvertFromJson<GitHubRunnerRegisterToken>(jsonResponse);
}
else
{
_term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'");
var errorResponse = await response.Content.ReadAsStringAsync();
_term.WriteError(errorResponse);
response.EnsureSuccessStatusCode();
return null;
}
}
} }
private async Task<GitHubAuthResult> GetTenantCredential(string githubUrl, string githubToken, string runnerEvent) private async Task<GitHubAuthResult> GetTenantCredential(string githubUrl, string githubToken, string runnerEvent)
{ {
var githubApiUrl = ""; var githubApiUrl = "";
var gitHubUrlBuilder = new UriBuilder(githubUrl); var gitHubUrlBuilder = new UriBuilder(githubUrl);
if (IsHostedServer(gitHubUrlBuilder)) if (UrlUtil.IsHostedServer(gitHubUrlBuilder))
{ {
githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/actions/runner-registration"; githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/actions/runner-registration";
} }

View File

@@ -13,7 +13,7 @@ namespace GitHub.Runner.Listener.Configuration
public interface ICredentialManager : IRunnerService public interface ICredentialManager : IRunnerService
{ {
ICredentialProvider GetCredentialProvider(string credType); ICredentialProvider GetCredentialProvider(string credType);
VssCredentials LoadCredentials(bool preferMigrated = true); VssCredentials LoadCredentials();
} }
public class CredentialManager : RunnerService, ICredentialManager public class CredentialManager : RunnerService, ICredentialManager
@@ -40,7 +40,7 @@ namespace GitHub.Runner.Listener.Configuration
return creds; return creds;
} }
public VssCredentials LoadCredentials(bool preferMigrated = true) public VssCredentials LoadCredentials()
{ {
IConfigurationStore store = HostContext.GetService<IConfigurationStore>(); IConfigurationStore store = HostContext.GetService<IConfigurationStore>();
@@ -50,14 +50,16 @@ namespace GitHub.Runner.Listener.Configuration
} }
CredentialData credData = store.GetCredentials(); CredentialData credData = store.GetCredentials();
var migratedCred = store.GetMigratedCredentials();
if (preferMigrated) if (migratedCred != null)
{ {
var migratedCred = store.GetMigratedCredentials(); credData = migratedCred;
if (migratedCred != null)
{ // Re-write .credentials with Token URL
credData = migratedCred; store.SaveCredential(credData);
}
// Delete .credentials_migrated
store.DeleteMigratedCredential();
} }
ICredentialProvider credProv = GetCredentialProvider(credData.Scheme); ICredentialProvider credProv = GetCredentialProvider(credData.Scheme);
@@ -69,6 +71,16 @@ namespace GitHub.Runner.Listener.Configuration
} }
} }
[DataContract]
public sealed class GitHubRunnerRegisterToken
{
[DataMember(Name = "token")]
public string Token { get; set; }
[DataMember(Name = "expires_at")]
public string ExpiresAt { get; set; }
}
[DataContract] [DataContract]
public sealed class GitHubAuthResult public sealed class GitHubAuthResult
{ {

View File

@@ -20,7 +20,7 @@ namespace GitHub.Runner.Listener.Configuration
/// key is returned to the caller. /// key is returned to the caller.
/// </summary> /// </summary>
/// <returns>An <c>RSACryptoServiceProvider</c> instance representing the key for the runner</returns> /// <returns>An <c>RSACryptoServiceProvider</c> instance representing the key for the runner</returns>
RSACryptoServiceProvider CreateKey(); RSA CreateKey();
/// <summary> /// <summary>
/// Deletes the RSA key managed by the key manager. /// Deletes the RSA key managed by the key manager.
@@ -32,7 +32,7 @@ namespace GitHub.Runner.Listener.Configuration
/// </summary> /// </summary>
/// <returns>An <c>RSACryptoServiceProvider</c> instance representing the key for the runner</returns> /// <returns>An <c>RSACryptoServiceProvider</c> instance representing the key for the runner</returns>
/// <exception cref="CryptographicException">No key exists in the store</exception> /// <exception cref="CryptographicException">No key exists in the store</exception>
RSACryptoServiceProvider GetKey(); RSA GetKey();
} }
// Newtonsoft 10 is not working properly with dotnet RSAParameters class // Newtonsoft 10 is not working properly with dotnet RSAParameters class

View File

@@ -36,7 +36,7 @@ namespace GitHub.Runner.Listener.Configuration
// We expect the key to be in the machine store at this point. Configuration should have set all of // We expect the key to be in the machine store at this point. Configuration should have set all of
// this up correctly so we can use the key to generate access tokens. // this up correctly so we can use the key to generate access tokens.
var keyManager = context.GetService<IRSAKeyManager>(); var keyManager = context.GetService<IRSAKeyManager>();
var signingCredentials = VssSigningCredentials.Create(() => keyManager.GetKey()); var signingCredentials = VssSigningCredentials.Create(() => keyManager.GetKey(), StringUtil.ConvertToBoolean(CredentialData.Data.GetValueOrDefault("requireFipsCryptography"), false));
var clientCredential = new VssOAuthJwtBearerClientCredential(clientId, authorizationUrl, signingCredentials); var clientCredential = new VssOAuthJwtBearerClientCredential(clientId, authorizationUrl, signingCredentials);
var agentCredential = new VssOAuthCredential(new Uri(oauthEndpointUrl, UriKind.Absolute), VssOAuthGrant.ClientCredentials, clientCredential); var agentCredential = new VssOAuthCredential(new Uri(oauthEndpointUrl, UriKind.Absolute), VssOAuthGrant.ClientCredentials, clientCredential);

View File

@@ -13,14 +13,14 @@ namespace GitHub.Runner.Listener.Configuration
private string _keyFile; private string _keyFile;
private IHostContext _context; private IHostContext _context;
public RSACryptoServiceProvider CreateKey() public RSA CreateKey()
{ {
RSACryptoServiceProvider rsa = null; RSA rsa = null;
if (!File.Exists(_keyFile)) if (!File.Exists(_keyFile))
{ {
Trace.Info("Creating new RSA key using 2048-bit key length"); Trace.Info("Creating new RSA key using 2048-bit key length");
rsa = new RSACryptoServiceProvider(2048); rsa = RSA.Create(2048);
// Now write the parameters to disk // Now write the parameters to disk
SaveParameters(rsa.ExportParameters(true)); SaveParameters(rsa.ExportParameters(true));
@@ -30,7 +30,7 @@ namespace GitHub.Runner.Listener.Configuration
{ {
Trace.Info("Found existing RSA key parameters file {0}", _keyFile); Trace.Info("Found existing RSA key parameters file {0}", _keyFile);
rsa = new RSACryptoServiceProvider(); rsa = RSA.Create();
rsa.ImportParameters(LoadParameters()); rsa.ImportParameters(LoadParameters());
} }
@@ -46,7 +46,7 @@ namespace GitHub.Runner.Listener.Configuration
} }
} }
public RSACryptoServiceProvider GetKey() public RSA GetKey()
{ {
if (!File.Exists(_keyFile)) if (!File.Exists(_keyFile))
{ {
@@ -55,7 +55,7 @@ namespace GitHub.Runner.Listener.Configuration
Trace.Info("Loading RSA key parameters from file {0}", _keyFile); Trace.Info("Loading RSA key parameters from file {0}", _keyFile);
var rsa = new RSACryptoServiceProvider(); var rsa = RSA.Create();
rsa.ImportParameters(LoadParameters()); rsa.ImportParameters(LoadParameters());
return rsa; return rsa;
} }

View File

@@ -14,14 +14,14 @@ namespace GitHub.Runner.Listener.Configuration
private string _keyFile; private string _keyFile;
private IHostContext _context; private IHostContext _context;
public RSACryptoServiceProvider CreateKey() public RSA CreateKey()
{ {
RSACryptoServiceProvider rsa = null; RSA rsa = null;
if (!File.Exists(_keyFile)) if (!File.Exists(_keyFile))
{ {
Trace.Info("Creating new RSA key using 2048-bit key length"); Trace.Info("Creating new RSA key using 2048-bit key length");
rsa = new RSACryptoServiceProvider(2048); rsa = RSA.Create(2048);
// Now write the parameters to disk // Now write the parameters to disk
IOUtil.SaveObject(new RSAParametersSerializable(rsa.ExportParameters(true)), _keyFile); IOUtil.SaveObject(new RSAParametersSerializable(rsa.ExportParameters(true)), _keyFile);
@@ -54,7 +54,7 @@ namespace GitHub.Runner.Listener.Configuration
{ {
Trace.Info("Found existing RSA key parameters file {0}", _keyFile); Trace.Info("Found existing RSA key parameters file {0}", _keyFile);
rsa = new RSACryptoServiceProvider(); rsa = RSA.Create();
rsa.ImportParameters(IOUtil.LoadObject<RSAParametersSerializable>(_keyFile).RSAParameters); rsa.ImportParameters(IOUtil.LoadObject<RSAParametersSerializable>(_keyFile).RSAParameters);
} }
@@ -70,7 +70,7 @@ namespace GitHub.Runner.Listener.Configuration
} }
} }
public RSACryptoServiceProvider GetKey() public RSA GetKey()
{ {
if (!File.Exists(_keyFile)) if (!File.Exists(_keyFile))
{ {
@@ -80,7 +80,7 @@ namespace GitHub.Runner.Listener.Configuration
Trace.Info("Loading RSA key parameters from file {0}", _keyFile); Trace.Info("Loading RSA key parameters from file {0}", _keyFile);
var parameters = IOUtil.LoadObject<RSAParametersSerializable>(_keyFile).RSAParameters; var parameters = IOUtil.LoadObject<RSAParametersSerializable>(_keyFile).RSAParameters;
var rsa = new RSACryptoServiceProvider(); var rsa = RSA.Create();
rsa.ImportParameters(parameters); rsa.ImportParameters(parameters);
return rsa; return rsa;
} }

View File

@@ -13,10 +13,7 @@ using System.Diagnostics;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using GitHub.Runner.Common; using GitHub.Runner.Common;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;
using GitHub.Services.WebApi;
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("Test")]
namespace GitHub.Runner.Listener namespace GitHub.Runner.Listener
{ {
[ServiceLocator(Default = typeof(MessageListener))] [ServiceLocator(Default = typeof(MessageListener))]
@@ -35,30 +32,18 @@ namespace GitHub.Runner.Listener
private ITerminal _term; private ITerminal _term;
private IRunnerServer _runnerServer; private IRunnerServer _runnerServer;
private TaskAgentSession _session; private TaskAgentSession _session;
private ICredentialManager _credMgr;
private IConfigurationStore _configStore;
private TimeSpan _getNextMessageRetryInterval; private TimeSpan _getNextMessageRetryInterval;
private readonly TimeSpan _sessionCreationRetryInterval = TimeSpan.FromSeconds(30); private readonly TimeSpan _sessionCreationRetryInterval = TimeSpan.FromSeconds(30);
private readonly TimeSpan _sessionConflictRetryLimit = TimeSpan.FromMinutes(4); private readonly TimeSpan _sessionConflictRetryLimit = TimeSpan.FromMinutes(4);
private readonly TimeSpan _clockSkewRetryLimit = TimeSpan.FromMinutes(30); private readonly TimeSpan _clockSkewRetryLimit = TimeSpan.FromMinutes(30);
private readonly Dictionary<string, int> _sessionCreationExceptionTracker = new Dictionary<string, int>(); private readonly Dictionary<string, int> _sessionCreationExceptionTracker = new Dictionary<string, int>();
// Whether load credentials from .credentials_migrated file
internal bool _useMigratedCredentials;
// need to check auth url if there is only .credentials and auth schema is OAuth
internal bool _needToCheckAuthorizationUrlUpdate;
internal Task<VssCredentials> _authorizationUrlMigrationBackgroundTask;
internal Task _authorizationUrlRollbackReattemptDelayBackgroundTask;
public override void Initialize(IHostContext hostContext) public override void Initialize(IHostContext hostContext)
{ {
base.Initialize(hostContext); base.Initialize(hostContext);
_term = HostContext.GetService<ITerminal>(); _term = HostContext.GetService<ITerminal>();
_runnerServer = HostContext.GetService<IRunnerServer>(); _runnerServer = HostContext.GetService<IRunnerServer>();
_credMgr = HostContext.GetService<ICredentialManager>();
_configStore = HostContext.GetService<IConfigurationStore>();
} }
public async Task<Boolean> CreateSessionAsync(CancellationToken token) public async Task<Boolean> CreateSessionAsync(CancellationToken token)
@@ -73,8 +58,8 @@ namespace GitHub.Runner.Listener
// Create connection. // Create connection.
Trace.Info("Loading Credentials"); Trace.Info("Loading Credentials");
_useMigratedCredentials = !StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_SPSAUTHURL")); var credMgr = HostContext.GetService<ICredentialManager>();
VssCredentials creds = _credMgr.LoadCredentials(_useMigratedCredentials); VssCredentials creds = credMgr.LoadCredentials();
var agent = new TaskAgentReference var agent = new TaskAgentReference
{ {
@@ -89,17 +74,6 @@ namespace GitHub.Runner.Listener
string errorMessage = string.Empty; string errorMessage = string.Empty;
bool encounteringError = false; bool encounteringError = false;
var originalCreds = _configStore.GetCredentials();
var migratedCreds = _configStore.GetMigratedCredentials();
if (migratedCreds == null)
{
_useMigratedCredentials = false;
if (originalCreds.Scheme == Constants.Configuration.OAuth)
{
_needToCheckAuthorizationUrlUpdate = true;
}
}
while (true) while (true)
{ {
token.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested();
@@ -127,12 +101,6 @@ namespace GitHub.Runner.Listener
encounteringError = false; encounteringError = false;
} }
if (_needToCheckAuthorizationUrlUpdate)
{
// start background task try to get new authorization url
_authorizationUrlMigrationBackgroundTask = GetNewOAuthAuthorizationSetting(token);
}
return true; return true;
} }
catch (OperationCanceledException) when (token.IsCancellationRequested) catch (OperationCanceledException) when (token.IsCancellationRequested)
@@ -164,44 +132,10 @@ namespace GitHub.Runner.Listener
} }
} }
if (ex is TaskAgentSessionConflictException)
{
try
{
var newCred = await GetNewOAuthAuthorizationSetting(token, true);
if (newCred != null)
{
await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), newCred);
Trace.Info("Updated connection to use migrated credential for next CreateSession call.");
_useMigratedCredentials = true;
_authorizationUrlMigrationBackgroundTask = null;
_needToCheckAuthorizationUrlUpdate = false;
}
}
catch (Exception e)
{
Trace.Error("Fail to refresh connection with new authorization url.");
Trace.Error(e);
}
}
if (!IsSessionCreationExceptionRetriable(ex)) if (!IsSessionCreationExceptionRetriable(ex))
{ {
if (_useMigratedCredentials && !(ex is TaskAgentSessionConflictException)) _term.WriteError($"Failed to create session. {ex.Message}");
{ return false;
// migrated credentials might cause lose permission during permission check,
// we will force to use original credential and try again
_useMigratedCredentials = false;
var reattemptBackoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromHours(24), TimeSpan.FromHours(36));
_authorizationUrlRollbackReattemptDelayBackgroundTask = HostContext.Delay(reattemptBackoff, token); // retry migrated creds in 24-36 hours.
creds = _credMgr.LoadCredentials(false);
Trace.Error("Fallback to original credentials and try again.");
}
else
{
_term.WriteError($"Failed to create session. {ex.Message}");
return false;
}
} }
if (!encounteringError) //print the message only on the first error if (!encounteringError) //print the message only on the first error
@@ -262,51 +196,6 @@ namespace GitHub.Runner.Listener
encounteringError = false; encounteringError = false;
continuousError = 0; continuousError = 0;
} }
if (_needToCheckAuthorizationUrlUpdate &&
_authorizationUrlMigrationBackgroundTask?.IsCompleted == true)
{
if (HostContext.GetService<IJobDispatcher>().Busy ||
HostContext.GetService<ISelfUpdater>().Busy)
{
Trace.Info("Job or runner updates in progress, update credentials next time.");
}
else
{
try
{
var newCred = await _authorizationUrlMigrationBackgroundTask;
await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), newCred);
Trace.Info("Updated connection to use migrated credential for next GetMessage call.");
_useMigratedCredentials = true;
_authorizationUrlMigrationBackgroundTask = null;
_needToCheckAuthorizationUrlUpdate = false;
}
catch (Exception ex)
{
Trace.Error("Fail to refresh connection with new authorization url.");
Trace.Error(ex);
}
}
}
if (_authorizationUrlRollbackReattemptDelayBackgroundTask?.IsCompleted == true)
{
try
{
// we rolled back to use original creds about 2 days before, now it's a good time to try migrated creds again.
Trace.Info("Re-attempt to use migrated credential");
var migratedCreds = _credMgr.LoadCredentials();
await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), migratedCreds);
_useMigratedCredentials = true;
_authorizationUrlRollbackReattemptDelayBackgroundTask = null;
}
catch (Exception ex)
{
Trace.Error("Fail to refresh connection with new authorization url on rollback reattempt.");
Trace.Error(ex);
}
}
} }
catch (OperationCanceledException) when (token.IsCancellationRequested) catch (OperationCanceledException) when (token.IsCancellationRequested)
{ {
@@ -330,21 +219,7 @@ namespace GitHub.Runner.Listener
} }
else if (!IsGetNextMessageExceptionRetriable(ex)) else if (!IsGetNextMessageExceptionRetriable(ex))
{ {
if (_useMigratedCredentials) throw;
{
// migrated credentials might cause lose permission during permission check,
// we will force to use original credential and try again
_useMigratedCredentials = false;
var reattemptBackoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromHours(24), TimeSpan.FromHours(36));
_authorizationUrlRollbackReattemptDelayBackgroundTask = HostContext.Delay(reattemptBackoff, token); // retry migrated creds in 24-36 hours.
var originalCreds = _credMgr.LoadCredentials(false);
await _runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), originalCreds);
Trace.Error("Fallback to original credentials and try again.");
}
else
{
throw;
}
} }
else else
{ {
@@ -444,7 +319,8 @@ namespace GitHub.Runner.Listener
var keyManager = HostContext.GetService<IRSAKeyManager>(); var keyManager = HostContext.GetService<IRSAKeyManager>();
using (var rsa = keyManager.GetKey()) using (var rsa = keyManager.GetKey())
{ {
return aes.CreateDecryptor(rsa.Decrypt(_session.EncryptionKey.Value, RSAEncryptionPadding.OaepSHA1), message.IV); var padding = _session.UseFipsEncryption ? RSAEncryptionPadding.OaepSHA256 : RSAEncryptionPadding.OaepSHA1;
return aes.CreateDecryptor(rsa.Decrypt(_session.EncryptionKey.Value, padding), message.IV);
} }
} }
else else
@@ -536,94 +412,5 @@ namespace GitHub.Runner.Listener
return true; return true;
} }
} }
private async Task<VssCredentials> GetNewOAuthAuthorizationSetting(CancellationToken token, bool adhoc = false)
{
Trace.Info("Start checking oauth authorization url update.");
while (true)
{
try
{
var migratedAuthorizationUrl = await _runnerServer.GetRunnerAuthUrlAsync(_settings.PoolId, _settings.AgentId);
if (!string.IsNullOrEmpty(migratedAuthorizationUrl))
{
var credData = _configStore.GetCredentials();
var clientId = credData.Data.GetValueOrDefault("clientId", null);
var currentAuthorizationUrl = credData.Data.GetValueOrDefault("authorizationUrl", null);
Trace.Info($"Current authorization url: {currentAuthorizationUrl}, new authorization url: {migratedAuthorizationUrl}");
if (string.Equals(currentAuthorizationUrl, migratedAuthorizationUrl, StringComparison.OrdinalIgnoreCase))
{
// We don't need to update credentials.
Trace.Info("No needs to update authorization url");
if (adhoc)
{
return null;
}
else
{
await Task.Delay(TimeSpan.FromMilliseconds(-1), token);
}
}
var keyManager = HostContext.GetService<IRSAKeyManager>();
var signingCredentials = VssSigningCredentials.Create(() => keyManager.GetKey());
var migratedClientCredential = new VssOAuthJwtBearerClientCredential(clientId, migratedAuthorizationUrl, signingCredentials);
var migratedRunnerCredential = new VssOAuthCredential(new Uri(migratedAuthorizationUrl, UriKind.Absolute), VssOAuthGrant.ClientCredentials, migratedClientCredential);
Trace.Info("Try connect service with Token Service OAuth endpoint.");
var runnerServer = HostContext.CreateService<IRunnerServer>();
await runnerServer.ConnectAsync(new Uri(_settings.ServerUrl), migratedRunnerCredential);
await runnerServer.GetAgentPoolsAsync();
Trace.Info($"Successfully connected service with new authorization url.");
var migratedCredData = new CredentialData
{
Scheme = Constants.Configuration.OAuth,
Data =
{
{ "clientId", clientId },
{ "authorizationUrl", migratedAuthorizationUrl },
{ "oauthEndpointUrl", migratedAuthorizationUrl },
},
};
_configStore.SaveMigratedCredential(migratedCredData);
return migratedRunnerCredential;
}
else
{
Trace.Verbose("No authorization url updates");
}
}
catch (Exception ex) when (!token.IsCancellationRequested)
{
Trace.Error("Fail to get/test new authorization url.");
Trace.Error(ex);
try
{
await _runnerServer.ReportRunnerAuthUrlErrorAsync(_settings.PoolId, _settings.AgentId, ex.ToString());
}
catch (Exception e)
{
// best effort
Trace.Error("Fail to report the migration error");
Trace.Error(e);
}
}
if (adhoc)
{
return null;
}
else
{
var backoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromMinutes(30), TimeSpan.FromMinutes(45));
await HostContext.Delay(backoff, token);
}
}
}
} }
} }

View File

@@ -102,7 +102,9 @@ namespace GitHub.Runner.Listener
IRunner runner = context.GetService<IRunner>(); IRunner runner = context.GetService<IRunner>();
try try
{ {
return await runner.ExecuteCommand(command); var returnCode = await runner.ExecuteCommand(command);
trace.Info($"Runner execution has finished with return code {returnCode}");
return returnCode;
} }
catch (OperationCanceledException) when (context.RunnerShutdownToken.IsCancellationRequested) catch (OperationCanceledException) when (context.RunnerShutdownToken.IsCancellationRequested)
{ {

View File

@@ -1,6 +1,5 @@
using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Listener.Configuration; using GitHub.Runner.Listener.Configuration;
using GitHub.Runner.Common.Util;
using System; using System;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
@@ -11,6 +10,8 @@ using System.Reflection;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using GitHub.Runner.Common; using GitHub.Runner.Common;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;
using System.Linq;
using GitHub.Runner.Listener.Check;
namespace GitHub.Runner.Listener namespace GitHub.Runner.Listener
{ {
@@ -72,6 +73,46 @@ namespace GitHub.Runner.Listener
return Constants.Runner.ReturnCode.Success; return Constants.Runner.ReturnCode.Success;
} }
if (command.Check)
{
var url = command.GetUrl();
var pat = command.GetGitHubPersonalAccessToken(required: true);
var checkExtensions = HostContext.GetService<IExtensionManager>().GetExtensions<ICheckExtension>();
var sortedChecks = checkExtensions.OrderBy(x => x.Order);
foreach (var check in sortedChecks)
{
_term.WriteLine($"**********************************************************************************************************************");
_term.WriteLine($"** Check: {check.CheckName}");
_term.WriteLine($"** Description: {check.CheckDescription}");
_term.WriteLine($"**********************************************************************************************************************");
var result = await check.RunCheck(url, pat);
if (!result)
{
_term.WriteLine($"** **");
_term.WriteLine($"** F A I L **");
_term.WriteLine($"** **");
_term.WriteLine($"**********************************************************************************************************************");
_term.WriteLine($"** Log: {check.CheckLog}");
_term.WriteLine($"** Help Doc: {check.HelpLink}");
_term.WriteLine($"**********************************************************************************************************************");
}
else
{
_term.WriteLine($"** **");
_term.WriteLine($"** P A S S **");
_term.WriteLine($"** **");
_term.WriteLine($"**********************************************************************************************************************");
_term.WriteLine($"** Log: {check.CheckLog}");
_term.WriteLine($"**********************************************************************************************************************");
}
_term.WriteLine();
_term.WriteLine();
}
return Constants.Runner.ReturnCode.Success;
}
// Configure runner prompt for args if not supplied // Configure runner prompt for args if not supplied
// Unattended configure mode will not prompt for args if not supplied and error on any missing or invalid value. // Unattended configure mode will not prompt for args if not supplied and error on any missing or invalid value.
if (command.Configure) if (command.Configure)
@@ -462,13 +503,14 @@ Options:
--commit Prints the runner commit --commit Prints the runner commit
Config Options: Config Options:
--unattended Disable interactive prompts for missing arguments. Defaults will be used for missing options --unattended Disable interactive prompts for missing arguments. Defaults will be used for missing options
--url string Repository to add the runner to. Required if unattended --url string Repository to add the runner to. Required if unattended
--token string Registration token. Required if unattended --token string Registration token. Required if unattended
--name string Name of the runner to configure (default {Environment.MachineName ?? "myrunner"}) --name string Name of the runner to configure (default {Environment.MachineName ?? "myrunner"})
--labels string Extra labels in addition to the default: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}' --runnergroup string Name of the runner group to add this runner to (defaults to the default runner group)
--work string Relative runner work directory (default {Constants.Path.WorkDirectory}) --labels string Extra labels in addition to the default: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}'
--replace Replace any existing runner with the same name (default false)"); --work string Relative runner work directory (default {Constants.Path.WorkDirectory})
--replace Replace any existing runner with the same name (default false)");
#if OS_WINDOWS #if OS_WINDOWS
_term.WriteLine($@" --runasservice Run the runner as a service"); _term.WriteLine($@" --runasservice Run the runner as a service");
_term.WriteLine($@" --windowslogonaccount string Account to run the service as. Requires runasservice"); _term.WriteLine($@" --windowslogonaccount string Account to run the service as. Requires runasservice");

View File

@@ -82,7 +82,7 @@ namespace GitHub.Runner.Plugins.Repository.v1_0
executionContext.Output($"Syncing repository: {repoFullName}"); executionContext.Output($"Syncing repository: {repoFullName}");
// Repository URL // Repository URL
var githubUrl = executionContext.GetGitHubContext("url"); var githubUrl = executionContext.GetGitHubContext("server_url");
var githubUri = new Uri(!string.IsNullOrEmpty(githubUrl) ? githubUrl : "https://github.com"); var githubUri = new Uri(!string.IsNullOrEmpty(githubUrl) ? githubUrl : "https://github.com");
var portInfo = githubUri.IsDefaultPort ? string.Empty : $":{githubUri.Port}"; var portInfo = githubUri.IsDefaultPort ? string.Empty : $":{githubUri.Port}";
Uri repositoryUrl = new Uri($"{githubUri.Scheme}://{githubUri.Host}{portInfo}/{repoFullName}"); Uri repositoryUrl = new Uri($"{githubUri.Scheme}://{githubUri.Host}{portInfo}/{repoFullName}");

View File

@@ -318,7 +318,12 @@ namespace GitHub.Runner.Sdk
} }
} }
using (var registration = cancellationToken.Register(async () => await CancelAndKillProcessTree(killProcessOnCancel))) var cancellationFinished = new TaskCompletionSource<bool>();
using (var registration = cancellationToken.Register(async () =>
{
await CancelAndKillProcessTree(killProcessOnCancel);
cancellationFinished.TrySetResult(true);
}))
{ {
Trace.Info($"Process started with process id {_proc.Id}, waiting for process exit."); Trace.Info($"Process started with process id {_proc.Id}, waiting for process exit.");
while (true) while (true)
@@ -341,6 +346,13 @@ namespace GitHub.Runner.Sdk
// data buffers one last time before returning // data buffers one last time before returning
ProcessOutput(); ProcessOutput();
if (cancellationToken.IsCancellationRequested)
{
// Ensure cancellation also finish on the cancellationToken.Register thread.
await cancellationFinished.Task;
Trace.Info($"Process Cancellation finished.");
}
Trace.Info($"Finished process {_proc.Id} with exit code {_proc.ExitCode}, and elapsed time {_stopWatch.Elapsed}."); Trace.Info($"Finished process {_proc.Id} with exit code {_proc.ExitCode}, and elapsed time {_stopWatch.Elapsed}.");
} }

View File

@@ -1,4 +1,4 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Net; using System.Net;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
@@ -71,7 +71,7 @@ namespace GitHub.Runner.Sdk
if (!string.IsNullOrEmpty(httpProxyAddress) && Uri.TryCreate(httpProxyAddress, UriKind.Absolute, out var proxyHttpUri)) if (!string.IsNullOrEmpty(httpProxyAddress) && Uri.TryCreate(httpProxyAddress, UriKind.Absolute, out var proxyHttpUri))
{ {
_httpProxyAddress = proxyHttpUri.AbsoluteUri; _httpProxyAddress = proxyHttpUri.OriginalString;
// Set both environment variables since there are tools support both casing (curl, wget) and tools support only one casing (docker) // Set both environment variables since there are tools support both casing (curl, wget) and tools support only one casing (docker)
Environment.SetEnvironmentVariable("HTTP_PROXY", _httpProxyAddress); Environment.SetEnvironmentVariable("HTTP_PROXY", _httpProxyAddress);
@@ -101,7 +101,7 @@ namespace GitHub.Runner.Sdk
if (!string.IsNullOrEmpty(httpsProxyAddress) && Uri.TryCreate(httpsProxyAddress, UriKind.Absolute, out var proxyHttpsUri)) if (!string.IsNullOrEmpty(httpsProxyAddress) && Uri.TryCreate(httpsProxyAddress, UriKind.Absolute, out var proxyHttpsUri))
{ {
_httpsProxyAddress = proxyHttpsUri.AbsoluteUri; _httpsProxyAddress = proxyHttpsUri.OriginalString;
// Set both environment variables since there are tools support both casing (curl, wget) and tools support only one casing (docker) // Set both environment variables since there are tools support both casing (curl, wget) and tools support only one casing (docker)
Environment.SetEnvironmentVariable("HTTPS_PROXY", _httpsProxyAddress); Environment.SetEnvironmentVariable("HTTPS_PROXY", _httpsProxyAddress);

View File

@@ -30,7 +30,7 @@ namespace GitHub.Runner.Sdk
// //
// For example, on an en-US box, this is required for loading the encoding for the // For example, on an en-US box, this is required for loading the encoding for the
// default console output code page '437'. Without loading the correct encoding for // default console output code page '437'. Without loading the correct encoding for
// code page IBM437, some characters cannot be translated correctly, e.g. write 'ç' // code page IBM437, some characters cannot be translated correctly, e.g. write 'ç'
// from powershell.exe. // from powershell.exe.
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
#endif #endif

View File

@@ -4,6 +4,13 @@ namespace GitHub.Runner.Sdk
{ {
public static class UrlUtil public static class UrlUtil
{ {
public static bool IsHostedServer(UriBuilder gitHubUrl)
{
return string.Equals(gitHubUrl.Host, "github.com", StringComparison.OrdinalIgnoreCase) ||
string.Equals(gitHubUrl.Host, "www.github.com", StringComparison.OrdinalIgnoreCase) ||
string.Equals(gitHubUrl.Host, "github.localhost", StringComparison.OrdinalIgnoreCase);
}
public static Uri GetCredentialEmbeddedUrl(Uri baseUrl, string username, string password) public static Uri GetCredentialEmbeddedUrl(Uri baseUrl, string username, string password)
{ {
ArgUtil.NotNull(baseUrl, nameof(baseUrl)); ArgUtil.NotNull(baseUrl, nameof(baseUrl));

View File

@@ -11,6 +11,11 @@ namespace GitHub.Runner.Sdk
{ {
ArgUtil.NotNullOrEmpty(command, nameof(command)); ArgUtil.NotNullOrEmpty(command, nameof(command));
trace?.Info($"Which: '{command}'"); trace?.Info($"Which: '{command}'");
if (Path.IsPathFullyQualified(command) && File.Exists(command))
{
trace?.Info($"Fully qualified path: '{command}'");
return command;
}
string path = Environment.GetEnvironmentVariable(PathUtil.PathVariable); string path = Environment.GetEnvironmentVariable(PathUtil.PathVariable);
if (string.IsNullOrEmpty(path)) if (string.IsNullOrEmpty(path))
{ {

View File

@@ -1,4 +1,5 @@
using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.Pipelines;
using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common.Util; using GitHub.Runner.Common.Util;
using GitHub.Runner.Worker.Container; using GitHub.Runner.Worker.Container;
@@ -183,12 +184,49 @@ namespace GitHub.Runner.Worker
public void ProcessCommand(IExecutionContext context, string line, ActionCommand command, ContainerInfo container) public void ProcessCommand(IExecutionContext context, string line, ActionCommand command, ContainerInfo container)
{ {
var allowUnsecureCommands = false;
bool.TryParse(Environment.GetEnvironmentVariable(Constants.Variables.Actions.AllowUnsupportedCommands), out allowUnsecureCommands);
// Apply environment from env context, env context contains job level env and action's env block
#if OS_WINDOWS
var envContext = context.ExpressionValues["env"] as DictionaryContextData;
#else
var envContext = context.ExpressionValues["env"] as CaseSensitiveDictionaryContextData;
#endif
if (!allowUnsecureCommands && envContext.ContainsKey(Constants.Variables.Actions.AllowUnsupportedCommands))
{
bool.TryParse(envContext[Constants.Variables.Actions.AllowUnsupportedCommands].ToString(), out allowUnsecureCommands);
}
if (!allowUnsecureCommands)
{
throw new Exception(String.Format(Constants.Runner.UnsupportedCommandMessageDisabled, this.Command));
}
if (!command.Properties.TryGetValue(SetEnvCommandProperties.Name, out string envName) || string.IsNullOrEmpty(envName)) if (!command.Properties.TryGetValue(SetEnvCommandProperties.Name, out string envName) || string.IsNullOrEmpty(envName))
{ {
throw new Exception("Required field 'name' is missing in ##[set-env] command."); throw new Exception("Required field 'name' is missing in ##[set-env] command.");
} }
context.EnvironmentVariables[envName] = command.Data;
foreach (var blocked in _setEnvBlockList)
{
if (string.Equals(blocked, envName, StringComparison.OrdinalIgnoreCase))
{
// Log Telemetry and let user know they shouldn't do this
var issue = new Issue()
{
Type = IssueType.Error,
Message = $"Can't update {blocked} environment variable using ::set-env:: command."
};
issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = $"{Constants.Runner.UnsupportedCommand}_{envName}";
context.AddIssue(issue);
return;
}
}
context.Global.EnvironmentVariables[envName] = command.Data;
context.SetEnvContext(envName, command.Data); context.SetEnvContext(envName, command.Data);
context.Debug($"{envName}='{command.Data}'"); context.Debug($"{envName}='{command.Data}'");
} }
@@ -197,6 +235,11 @@ namespace GitHub.Runner.Worker
{ {
public const String Name = "name"; public const String Name = "name";
} }
private string[] _setEnvBlockList =
{
"NODE_OPTIONS"
};
} }
public sealed class SetOutputCommandExtension : RunnerService, IActionCommandExtension public sealed class SetOutputCommandExtension : RunnerService, IActionCommandExtension
@@ -281,10 +324,29 @@ namespace GitHub.Runner.Worker
public Type ExtensionType => typeof(IActionCommandExtension); public Type ExtensionType => typeof(IActionCommandExtension);
public void ProcessCommand(IExecutionContext context, string line, ActionCommand command, ContainerInfo container) public void ProcessCommand(IExecutionContext context, string line, ActionCommand command, ContainerInfo container)
{ {
var allowUnsecureCommands = false;
bool.TryParse(Environment.GetEnvironmentVariable(Constants.Variables.Actions.AllowUnsupportedCommands), out allowUnsecureCommands);
// Apply environment from env context, env context contains job level env and action's env block
#if OS_WINDOWS
var envContext = context.ExpressionValues["env"] as DictionaryContextData;
#else
var envContext = context.ExpressionValues["env"] as CaseSensitiveDictionaryContextData;
#endif
if (!allowUnsecureCommands && envContext.ContainsKey(Constants.Variables.Actions.AllowUnsupportedCommands))
{
bool.TryParse(envContext[Constants.Variables.Actions.AllowUnsupportedCommands].ToString(), out allowUnsecureCommands);
}
if (!allowUnsecureCommands)
{
throw new Exception(String.Format(Constants.Runner.UnsupportedCommandMessageDisabled, this.Command));
}
ArgUtil.NotNullOrEmpty(command.Data, "path"); ArgUtil.NotNullOrEmpty(command.Data, "path");
context.PrependPath.RemoveAll(x => string.Equals(x, command.Data, StringComparison.CurrentCulture)); context.Global.PrependPath.RemoveAll(x => string.Equals(x, command.Data, StringComparison.CurrentCulture));
context.PrependPath.Add(command.Data); context.Global.PrependPath.Add(command.Data);
} }
} }

View File

@@ -14,6 +14,7 @@ using GitHub.Runner.Common;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;
using GitHub.Runner.Worker.Container; using GitHub.Runner.Worker.Container;
using GitHub.Services.Common; using GitHub.Services.Common;
using WebApi = GitHub.DistributedTask.WebApi;
using Pipelines = GitHub.DistributedTask.Pipelines; using Pipelines = GitHub.DistributedTask.Pipelines;
using PipelineTemplateConstants = GitHub.DistributedTask.Pipelines.ObjectTemplating.PipelineTemplateConstants; using PipelineTemplateConstants = GitHub.DistributedTask.Pipelines.ObjectTemplating.PipelineTemplateConstants;
@@ -65,7 +66,7 @@ namespace GitHub.Runner.Worker
// TODO: Deprecate the PREVIEW_ACTION_TOKEN // TODO: Deprecate the PREVIEW_ACTION_TOKEN
// Log even if we aren't using it to ensure users know. // Log even if we aren't using it to ensure users know.
if (!string.IsNullOrEmpty(executionContext.Variables.Get("PREVIEW_ACTION_TOKEN"))) if (!string.IsNullOrEmpty(executionContext.Global.Variables.Get("PREVIEW_ACTION_TOKEN")))
{ {
executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is deprecated. Please remove it from the repository's secrets"); executionContext.Warning("The 'PREVIEW_ACTION_TOKEN' secret is deprecated. Please remove it from the repository's secrets");
} }
@@ -73,6 +74,11 @@ namespace GitHub.Runner.Worker
// Clear the cache (for self-hosted runners) // Clear the cache (for self-hosted runners)
IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken); IOUtil.DeleteDirectory(HostContext.GetDirectory(WellKnownDirectory.Actions), executionContext.CancellationToken);
// todo: Remove when feature flag DistributedTask.NewActionMetadata is removed
var newActionMetadata = executionContext.Global.Variables.GetBoolean("DistributedTask.NewActionMetadata") ?? false;
var repositoryActions = new List<Pipelines.ActionStep>();
foreach (var action in actions) foreach (var action in actions)
{ {
if (action.Reference.Type == Pipelines.ActionSourceType.ContainerRegistry) if (action.Reference.Type == Pipelines.ActionSourceType.ContainerRegistry)
@@ -90,7 +96,8 @@ namespace GitHub.Runner.Worker
Trace.Info($"Action {action.Name} ({action.Id}) needs to pull image '{containerReference.Image}'"); Trace.Info($"Action {action.Name} ({action.Id}) needs to pull image '{containerReference.Image}'");
imagesToPull[containerReference.Image].Add(action.Id); imagesToPull[containerReference.Image].Add(action.Id);
} }
else if (action.Reference.Type == Pipelines.ActionSourceType.Repository) // todo: Remove when feature flag DistributedTask.NewActionMetadata is removed
else if (action.Reference.Type == Pipelines.ActionSourceType.Repository && !newActionMetadata)
{ {
// only download the repository archive // only download the repository archive
await DownloadRepositoryActionAsync(executionContext, action); await DownloadRepositoryActionAsync(executionContext, action);
@@ -124,6 +131,81 @@ namespace GitHub.Runner.Worker
} }
} }
var repoAction = action.Reference as Pipelines.RepositoryPathReference;
if (repoAction.RepositoryType != Pipelines.PipelineConstants.SelfAlias)
{
var definition = LoadAction(executionContext, action);
if (definition.Data.Execution.HasPre)
{
var actionRunner = HostContext.CreateService<IActionRunner>();
actionRunner.Action = action;
actionRunner.Stage = ActionRunStage.Pre;
actionRunner.Condition = definition.Data.Execution.InitCondition;
Trace.Info($"Add 'pre' execution for {action.Id}");
preStepTracker[action.Id] = actionRunner;
}
}
}
else if (action.Reference.Type == Pipelines.ActionSourceType.Repository && newActionMetadata)
{
repositoryActions.Add(action);
}
}
if (repositoryActions.Count > 0)
{
// Get the download info
var downloadInfos = await GetDownloadInfoAsync(executionContext, repositoryActions);
// Download each action
foreach (var action in repositoryActions)
{
var lookupKey = GetDownloadInfoLookupKey(action);
if (string.IsNullOrEmpty(lookupKey))
{
continue;
}
if (!downloadInfos.TryGetValue(lookupKey, out var downloadInfo))
{
throw new Exception($"Missing download info for {lookupKey}");
}
await DownloadRepositoryActionAsync(executionContext, downloadInfo);
}
// More preparation based on content in the repository (action.yml)
foreach (var action in repositoryActions)
{
var setupInfo = PrepareRepositoryActionAsync(executionContext, action);
if (setupInfo != null)
{
if (!string.IsNullOrEmpty(setupInfo.Image))
{
if (!imagesToPull.ContainsKey(setupInfo.Image))
{
imagesToPull[setupInfo.Image] = new List<Guid>();
}
Trace.Info($"Action {action.Name} ({action.Id}) from repository '{setupInfo.ActionRepository}' needs to pull image '{setupInfo.Image}'");
imagesToPull[setupInfo.Image].Add(action.Id);
}
else
{
ArgUtil.NotNullOrEmpty(setupInfo.ActionRepository, nameof(setupInfo.ActionRepository));
if (!imagesToBuild.ContainsKey(setupInfo.ActionRepository))
{
imagesToBuild[setupInfo.ActionRepository] = new List<Guid>();
}
Trace.Info($"Action {action.Name} ({action.Id}) from repository '{setupInfo.ActionRepository}' needs to build image '{setupInfo.Dockerfile}'");
imagesToBuild[setupInfo.ActionRepository].Add(action.Id);
imagesToBuildInfo[setupInfo.ActionRepository] = setupInfo;
}
}
var repoAction = action.Reference as Pipelines.RepositoryPathReference; var repoAction = action.Reference as Pipelines.RepositoryPathReference;
if (repoAction.RepositoryType != Pipelines.PipelineConstants.SelfAlias) if (repoAction.RepositoryType != Pipelines.PipelineConstants.SelfAlias)
{ {
@@ -313,6 +395,14 @@ namespace GitHub.Runner.Worker
Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}."); Trace.Info($"Action cleanup plugin: {plugin.PluginTypeName}.");
} }
} }
else if (definition.Data.Execution.ExecutionType == ActionExecutionType.Composite)
{
var compositeAction = definition.Data.Execution as CompositeActionExecutionData;
Trace.Info($"Load {compositeAction.Steps?.Count ?? 0} action steps.");
Trace.Verbose($"Details: {StringUtil.ConvertToJson(compositeAction?.Steps)}");
Trace.Info($"Load: {compositeAction.Outputs?.Count ?? 0} number of outputs");
Trace.Info($"Details: {StringUtil.ConvertToJson(compositeAction?.Outputs)}");
}
else else
{ {
throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString()); throw new NotSupportedException(definition.Data.Execution.ExecutionType.ToString());
@@ -378,7 +468,7 @@ namespace GitHub.Runner.Worker
ArgUtil.NotNull(setupInfo, nameof(setupInfo)); ArgUtil.NotNull(setupInfo, nameof(setupInfo));
ArgUtil.NotNullOrEmpty(setupInfo.Container.Image, nameof(setupInfo.Container.Image)); ArgUtil.NotNullOrEmpty(setupInfo.Container.Image, nameof(setupInfo.Container.Image));
executionContext.Output($"Pull down action image '{setupInfo.Container.Image}'"); executionContext.Output($"##[group]Pull down action image '{setupInfo.Container.Image}'");
// Pull down docker image with retry up to 3 times // Pull down docker image with retry up to 3 times
var dockerManger = HostContext.GetService<IDockerCommandManager>(); var dockerManger = HostContext.GetService<IDockerCommandManager>();
@@ -402,6 +492,7 @@ namespace GitHub.Runner.Worker
} }
} }
} }
executionContext.Output("##[endgroup]");
if (retryCount == 3 && pullExitCode != 0) if (retryCount == 3 && pullExitCode != 0)
{ {
@@ -421,7 +512,7 @@ namespace GitHub.Runner.Worker
ArgUtil.NotNull(setupInfo, nameof(setupInfo)); ArgUtil.NotNull(setupInfo, nameof(setupInfo));
ArgUtil.NotNullOrEmpty(setupInfo.Container.Dockerfile, nameof(setupInfo.Container.Dockerfile)); ArgUtil.NotNullOrEmpty(setupInfo.Container.Dockerfile, nameof(setupInfo.Container.Dockerfile));
executionContext.Output($"Build container for action use: '{setupInfo.Container.Dockerfile}'."); executionContext.Output($"##[group]Build container for action use: '{setupInfo.Container.Dockerfile}'.");
// Build docker image with retry up to 3 times // Build docker image with retry up to 3 times
var dockerManger = HostContext.GetService<IDockerCommandManager>(); var dockerManger = HostContext.GetService<IDockerCommandManager>();
@@ -451,6 +542,7 @@ namespace GitHub.Runner.Worker
} }
} }
} }
executionContext.Output("##[endgroup]");
if (retryCount == 3 && buildExitCode != 0) if (retryCount == 3 && buildExitCode != 0)
{ {
@@ -464,6 +556,98 @@ namespace GitHub.Runner.Worker
} }
} }
// This implementation is temporary and will be replaced with a REST API call to the service to resolve
private async Task<IDictionary<string, WebApi.ActionDownloadInfo>> GetDownloadInfoAsync(IExecutionContext executionContext, List<Pipelines.ActionStep> actions)
{
executionContext.Output("Getting action download info");
// Convert to action reference
var actionReferences = actions
.GroupBy(x => GetDownloadInfoLookupKey(x))
.Where(x => !string.IsNullOrEmpty(x.Key))
.Select(x =>
{
var action = x.First();
var repositoryReference = action.Reference as Pipelines.RepositoryPathReference;
ArgUtil.NotNull(repositoryReference, nameof(repositoryReference));
return new WebApi.ActionReference
{
NameWithOwner = repositoryReference.Name,
Ref = repositoryReference.Ref,
};
})
.ToList();
// Nothing to resolve?
if (actionReferences.Count == 0)
{
return new Dictionary<string, WebApi.ActionDownloadInfo>();
}
// Resolve download info
var jobServer = HostContext.GetService<IJobServer>();
var actionDownloadInfos = default(WebApi.ActionDownloadInfoCollection);
for (var attempt = 1; attempt <= 3; attempt++)
{
try
{
actionDownloadInfos = await jobServer.ResolveActionDownloadInfoAsync(executionContext.Global.Plan.ScopeIdentifier, executionContext.Global.Plan.PlanType, executionContext.Global.Plan.PlanId, new WebApi.ActionReferenceList { Actions = actionReferences }, executionContext.CancellationToken);
break;
}
catch (Exception ex) when (!executionContext.CancellationToken.IsCancellationRequested) // Do not retry if the run is canceled.
{
if (attempt < 3)
{
executionContext.Output($"Failed to resolve action download info. Error: {ex.Message}");
executionContext.Debug(ex.ToString());
if (String.IsNullOrEmpty(Environment.GetEnvironmentVariable("_GITHUB_ACTION_DOWNLOAD_NO_BACKOFF")))
{
var backoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(30));
executionContext.Output($"Retrying in {backoff.TotalSeconds} seconds");
await Task.Delay(backoff);
}
}
else
{
// Some possible cases are:
// * Repo is rate limited
// * Repo or tag doesn't exist, or isn't public
if (ex is WebApi.UnresolvableActionDownloadInfoException)
{
throw;
}
else
{
// This exception will be traced as an infrastructure failure
throw new WebApi.FailedToResolveActionDownloadInfoException("Failed to resolve action download info.", ex);
}
}
}
}
ArgUtil.NotNull(actionDownloadInfos, nameof(actionDownloadInfos));
ArgUtil.NotNull(actionDownloadInfos.Actions, nameof(actionDownloadInfos.Actions));
var apiUrl = GetApiUrl(executionContext);
var defaultAccessToken = executionContext.GetGitHubContext("token");
var configurationStore = HostContext.GetService<IConfigurationStore>();
var runnerSettings = configurationStore.GetSettings();
foreach (var actionDownloadInfo in actionDownloadInfos.Actions.Values)
{
// Add secret
HostContext.SecretMasker.AddValue(actionDownloadInfo.Authentication?.Token);
// Default auth token
if (string.IsNullOrEmpty(actionDownloadInfo.Authentication?.Token))
{
actionDownloadInfo.Authentication = new WebApi.ActionDownloadAuthentication { Token = defaultAccessToken };
}
}
return actionDownloadInfos.Actions;
}
// todo: Remove when feature flag DistributedTask.NewActionMetadata is removed
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction) private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, Pipelines.ActionStep repositoryAction)
{ {
Trace.Entering(); Trace.Entering();
@@ -507,9 +691,8 @@ namespace GitHub.Runner.Worker
{ {
string apiUrl = GetApiUrl(executionContext); string apiUrl = GetApiUrl(executionContext);
string archiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref); string archiveLink = BuildLinkToActionArchive(apiUrl, repositoryReference.Name, repositoryReference.Ref);
Trace.Info($"Download archive '{archiveLink}' to '{destDirectory}'.");
var downloadDetails = new ActionDownloadDetails(archiveLink, ConfigureAuthorizationFromContext); var downloadDetails = new ActionDownloadDetails(archiveLink, ConfigureAuthorizationFromContext);
await DownloadRepositoryActionAsync(executionContext, downloadDetails, destDirectory); await DownloadRepositoryActionAsync(executionContext, downloadDetails, null, destDirectory);
return; return;
} }
else else
@@ -533,10 +716,9 @@ namespace GitHub.Runner.Worker
foreach (var downloadAttempt in downloadAttempts) foreach (var downloadAttempt in downloadAttempts)
{ {
Trace.Info($"Download archive '{downloadAttempt.ArchiveLink}' to '{destDirectory}'.");
try try
{ {
await DownloadRepositoryActionAsync(executionContext, downloadAttempt, destDirectory); await DownloadRepositoryActionAsync(executionContext, downloadAttempt, null, destDirectory);
return; return;
} }
catch (ActionNotFoundException) catch (ActionNotFoundException)
@@ -549,6 +731,32 @@ namespace GitHub.Runner.Worker
} }
} }
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, WebApi.ActionDownloadInfo downloadInfo)
{
Trace.Entering();
ArgUtil.NotNull(executionContext, nameof(executionContext));
ArgUtil.NotNull(downloadInfo, nameof(downloadInfo));
ArgUtil.NotNullOrEmpty(downloadInfo.NameWithOwner, nameof(downloadInfo.NameWithOwner));
ArgUtil.NotNullOrEmpty(downloadInfo.Ref, nameof(downloadInfo.Ref));
string destDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), downloadInfo.NameWithOwner.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar), downloadInfo.Ref);
string watermarkFile = GetWatermarkFilePath(destDirectory);
if (File.Exists(watermarkFile))
{
executionContext.Debug($"Action '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}' already downloaded at '{destDirectory}'.");
return;
}
else
{
// make sure we get a clean folder ready to use.
IOUtil.DeleteDirectory(destDirectory, executionContext.CancellationToken);
Directory.CreateDirectory(destDirectory);
executionContext.Output($"Download action repository '{downloadInfo.NameWithOwner}@{downloadInfo.Ref}'");
}
await DownloadRepositoryActionAsync(executionContext, null, downloadInfo, destDirectory);
}
private string GetApiUrl(IExecutionContext executionContext) private string GetApiUrl(IExecutionContext executionContext)
{ {
string apiUrl = executionContext.GetGitHubContext("api_url"); string apiUrl = executionContext.GetGitHubContext("api_url");
@@ -569,7 +777,8 @@ namespace GitHub.Runner.Worker
#endif #endif
} }
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadDetails actionDownloadDetails, string destDirectory) // todo: Remove the parameter "actionDownloadDetails" when feature flag DistributedTask.NewActionMetadata is removed
private async Task DownloadRepositoryActionAsync(IExecutionContext executionContext, ActionDownloadDetails actionDownloadDetails, WebApi.ActionDownloadInfo downloadInfo, string destDirectory)
{ {
//download and extract action in a temp folder and rename it on success //download and extract action in a temp folder and rename it on success
string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid()); string tempDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Actions), "_temp_" + Guid.NewGuid());
@@ -577,11 +786,12 @@ namespace GitHub.Runner.Worker
#if OS_WINDOWS #if OS_WINDOWS
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip"); string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.zip");
string link = downloadInfo?.ZipballUrl ?? actionDownloadDetails.ArchiveLink;
#else #else
string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz"); string archiveFile = Path.Combine(tempDirectory, $"{Guid.NewGuid()}.tar.gz");
string link = downloadInfo?.TarballUrl ?? actionDownloadDetails.ArchiveLink;
#endif #endif
string link = actionDownloadDetails.ArchiveLink;
Trace.Info($"Save archive '{link}' into {archiveFile}."); Trace.Info($"Save archive '{link}' into {archiveFile}.");
try try
{ {
@@ -601,7 +811,16 @@ namespace GitHub.Runner.Worker
using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClientHandler = HostContext.CreateHttpClientHandler())
using (var httpClient = new HttpClient(httpClientHandler)) using (var httpClient = new HttpClient(httpClientHandler))
{ {
actionDownloadDetails.ConfigureAuthorization(executionContext, httpClient); // Legacy
if (downloadInfo == null)
{
actionDownloadDetails.ConfigureAuthorization(executionContext, httpClient);
}
// FF DistributedTask.NewActionMetadata
else
{
httpClient.DefaultRequestHeaders.Authorization = CreateAuthHeader(downloadInfo.Authentication?.Token);
}
httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents);
using (var response = await httpClient.GetAsync(link)) using (var response = await httpClient.GetAsync(link))
@@ -741,13 +960,14 @@ namespace GitHub.Runner.Worker
} }
} }
// todo: Remove when feature flag DistributedTask.NewActionMetadata is removed
private void ConfigureAuthorizationFromContext(IExecutionContext executionContext, HttpClient httpClient) private void ConfigureAuthorizationFromContext(IExecutionContext executionContext, HttpClient httpClient)
{ {
var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN"); var authToken = Environment.GetEnvironmentVariable("_GITHUB_ACTION_TOKEN");
if (string.IsNullOrEmpty(authToken)) if (string.IsNullOrEmpty(authToken))
{ {
// TODO: Deprecate the PREVIEW_ACTION_TOKEN // TODO: Deprecate the PREVIEW_ACTION_TOKEN
authToken = executionContext.Variables.Get("PREVIEW_ACTION_TOKEN"); authToken = executionContext.Global.Variables.Get("PREVIEW_ACTION_TOKEN");
} }
if (!string.IsNullOrEmpty(authToken)) if (!string.IsNullOrEmpty(authToken))
@@ -846,6 +1066,11 @@ namespace GitHub.Runner.Worker
Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation."); Trace.Info($"Action plugin: {(actionDefinitionData.Execution as PluginActionExecutionData).Plugin}, no more preparation.");
return null; return null;
} }
else if (actionDefinitionData.Execution.ExecutionType == ActionExecutionType.Composite)
{
Trace.Info($"Action composite: {(actionDefinitionData.Execution as CompositeActionExecutionData).Steps}, no more preparation.");
return null;
}
else else
{ {
throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString()); throw new NotSupportedException(actionDefinitionData.Execution.ExecutionType.ToString());
@@ -872,6 +1097,51 @@ namespace GitHub.Runner.Worker
} }
} }
private static string GetDownloadInfoLookupKey(Pipelines.ActionStep action)
{
if (action.Reference.Type != Pipelines.ActionSourceType.Repository)
{
return null;
}
var repositoryReference = action.Reference as Pipelines.RepositoryPathReference;
ArgUtil.NotNull(repositoryReference, nameof(repositoryReference));
if (string.Equals(repositoryReference.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
return null;
}
if (!string.Equals(repositoryReference.RepositoryType, Pipelines.RepositoryTypes.GitHub, StringComparison.OrdinalIgnoreCase))
{
throw new NotSupportedException(repositoryReference.RepositoryType);
}
ArgUtil.NotNullOrEmpty(repositoryReference.Name, nameof(repositoryReference.Name));
ArgUtil.NotNullOrEmpty(repositoryReference.Ref, nameof(repositoryReference.Ref));
return $"{repositoryReference.Name}@{repositoryReference.Ref}";
}
private static string GetDownloadInfoLookupKey(WebApi.ActionDownloadInfo info)
{
ArgUtil.NotNullOrEmpty(info.NameWithOwner, nameof(info.NameWithOwner));
ArgUtil.NotNullOrEmpty(info.Ref, nameof(info.Ref));
return $"{info.NameWithOwner}@{info.Ref}";
}
private AuthenticationHeaderValue CreateAuthHeader(string token)
{
if (string.IsNullOrEmpty(token))
{
return null;
}
var base64EncodingToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{token}"));
HostContext.SecretMasker.AddValue(base64EncodingToken);
return new AuthenticationHeaderValue("Basic", base64EncodingToken);
}
// todo: Remove when feature flag DistributedTask.NewActionMetadata is removed
private class ActionDownloadDetails private class ActionDownloadDetails
{ {
public string ArchiveLink { get; } public string ArchiveLink { get; }
@@ -911,6 +1181,7 @@ namespace GitHub.Runner.Worker
NodeJS, NodeJS,
Plugin, Plugin,
Script, Script,
Composite,
} }
public sealed class ContainerActionExecutionData : ActionExecutionData public sealed class ContainerActionExecutionData : ActionExecutionData
@@ -967,6 +1238,15 @@ namespace GitHub.Runner.Worker
public override bool HasPost => false; public override bool HasPost => false;
} }
public sealed class CompositeActionExecutionData : ActionExecutionData
{
public override ActionExecutionType ExecutionType => ActionExecutionType.Composite;
public override bool HasPre => false;
public override bool HasPost => false;
public List<Pipelines.ActionStep> Steps { get; set; }
public MappingToken Outputs { get; set; }
}
public abstract class ActionExecutionData public abstract class ActionExecutionData
{ {
private string _initCondition = $"{Constants.Expressions.Always}()"; private string _initCondition = $"{Constants.Expressions.Always}()";

View File

@@ -14,6 +14,7 @@ using YamlDotNet.Core;
using YamlDotNet.Core.Events; using YamlDotNet.Core.Events;
using System.Globalization; using System.Globalization;
using System.Linq; using System.Linq;
using Pipelines = GitHub.DistributedTask.Pipelines;
namespace GitHub.Runner.Worker namespace GitHub.Runner.Worker
{ {
@@ -22,6 +23,8 @@ namespace GitHub.Runner.Worker
{ {
ActionDefinitionData Load(IExecutionContext executionContext, string manifestFile); ActionDefinitionData Load(IExecutionContext executionContext, string manifestFile);
DictionaryContextData EvaluateCompositeOutputs(IExecutionContext executionContext, TemplateToken token, IDictionary<string, PipelineContextData> extraExpressionValues);
List<string> EvaluateContainerArguments(IExecutionContext executionContext, SequenceToken token, IDictionary<string, PipelineContextData> extraExpressionValues); List<string> EvaluateContainerArguments(IExecutionContext executionContext, SequenceToken token, IDictionary<string, PipelineContextData> extraExpressionValues);
Dictionary<string, string> EvaluateContainerEnvironment(IExecutionContext executionContext, MappingToken token, IDictionary<string, PipelineContextData> extraExpressionValues); Dictionary<string, string> EvaluateContainerEnvironment(IExecutionContext executionContext, MappingToken token, IDictionary<string, PipelineContextData> extraExpressionValues);
@@ -32,8 +35,6 @@ namespace GitHub.Runner.Worker
public sealed class ActionManifestManager : RunnerService, IActionManifestManager public sealed class ActionManifestManager : RunnerService, IActionManifestManager
{ {
private TemplateSchema _actionManifestSchema; private TemplateSchema _actionManifestSchema;
private IReadOnlyList<String> _fileTable;
public override void Initialize(IHostContext hostContext) public override void Initialize(IHostContext hostContext)
{ {
base.Initialize(hostContext); base.Initialize(hostContext);
@@ -54,25 +55,45 @@ namespace GitHub.Runner.Worker
public ActionDefinitionData Load(IExecutionContext executionContext, string manifestFile) public ActionDefinitionData Load(IExecutionContext executionContext, string manifestFile)
{ {
var context = CreateContext(executionContext); var templateContext = CreateTemplateContext(executionContext);
ActionDefinitionData actionDefinition = new ActionDefinitionData(); ActionDefinitionData actionDefinition = new ActionDefinitionData();
// Clean up file name real quick
// Instead of using Regex which can be computationally expensive,
// we can just remove the # of characters from the fileName according to the length of the basePath
string basePath = HostContext.GetDirectory(WellKnownDirectory.Actions);
string fileRelativePath = manifestFile;
if (manifestFile.Contains(basePath))
{
fileRelativePath = manifestFile.Remove(0, basePath.Length + 1);
}
try try
{ {
var token = default(TemplateToken); var token = default(TemplateToken);
// Get the file ID // Get the file ID
var fileId = context.GetFileId(manifestFile); var fileId = templateContext.GetFileId(fileRelativePath);
_fileTable = context.GetFileTable();
// Add this file to the FileTable in executionContext if it hasn't been added already
// we use > since fileID is 1 indexed
if (fileId > executionContext.Global.FileTable.Count)
{
executionContext.Global.FileTable.Add(fileRelativePath);
}
// Read the file // Read the file
var fileContent = File.ReadAllText(manifestFile); var fileContent = File.ReadAllText(manifestFile);
using (var stringReader = new StringReader(fileContent)) using (var stringReader = new StringReader(fileContent))
{ {
var yamlObjectReader = new YamlObjectReader(null, stringReader); var yamlObjectReader = new YamlObjectReader(fileId, stringReader);
token = TemplateReader.Read(context, "action-root", yamlObjectReader, fileId, out _); token = TemplateReader.Read(templateContext, "action-root", yamlObjectReader, fileId, out _);
} }
var actionMapping = token.AssertMapping("action manifest root"); var actionMapping = token.AssertMapping("action manifest root");
var actionOutputs = default(MappingToken);
var actionRunValueToken = default(TemplateToken);
foreach (var actionPair in actionMapping) foreach (var actionPair in actionMapping)
{ {
var propertyName = actionPair.Key.AssertString($"action.yml property key"); var propertyName = actionPair.Key.AssertString($"action.yml property key");
@@ -83,44 +104,56 @@ namespace GitHub.Runner.Worker
actionDefinition.Name = actionPair.Value.AssertString("name").Value; actionDefinition.Name = actionPair.Value.AssertString("name").Value;
break; break;
case "outputs":
actionOutputs = actionPair.Value.AssertMapping("outputs");
break;
case "description": case "description":
actionDefinition.Description = actionPair.Value.AssertString("description").Value; actionDefinition.Description = actionPair.Value.AssertString("description").Value;
break; break;
case "inputs": case "inputs":
ConvertInputs(context, actionPair.Value, actionDefinition); ConvertInputs(actionPair.Value, actionDefinition);
break; break;
case "runs": case "runs":
actionDefinition.Execution = ConvertRuns(context, actionPair.Value); // Defer runs token evaluation to after for loop to ensure that order of outputs doesn't matter.
actionRunValueToken = actionPair.Value;
break; break;
default: default:
Trace.Info($"Ignore action property {propertyName}."); Trace.Info($"Ignore action property {propertyName}.");
break; break;
} }
} }
// Evaluate Runs Last
if (actionRunValueToken != null)
{
actionDefinition.Execution = ConvertRuns(executionContext, templateContext, actionRunValueToken, fileRelativePath, actionOutputs);
}
} }
catch (Exception ex) catch (Exception ex)
{ {
Trace.Error(ex); Trace.Error(ex);
context.Errors.Add(ex); templateContext.Errors.Add(ex);
} }
if (context.Errors.Count > 0) if (templateContext.Errors.Count > 0)
{ {
foreach (var error in context.Errors) foreach (var error in templateContext.Errors)
{ {
Trace.Error($"Action.yml load error: {error.Message}"); Trace.Error($"Action.yml load error: {error.Message}");
executionContext.Error(error.Message); executionContext.Error(error.Message);
} }
throw new ArgumentException($"Fail to load {manifestFile}"); throw new ArgumentException($"Fail to load {fileRelativePath}");
} }
if (actionDefinition.Execution == null) if (actionDefinition.Execution == null)
{ {
executionContext.Debug($"Loaded action.yml file: {StringUtil.ConvertToJson(actionDefinition)}"); executionContext.Debug($"Loaded action.yml file: {StringUtil.ConvertToJson(actionDefinition)}");
throw new ArgumentException($"Top level 'runs:' section is required for {manifestFile}"); throw new ArgumentException($"Top level 'runs:' section is required for {fileRelativePath}");
} }
else else
{ {
@@ -130,6 +163,33 @@ namespace GitHub.Runner.Worker
return actionDefinition; return actionDefinition;
} }
public DictionaryContextData EvaluateCompositeOutputs(
IExecutionContext executionContext,
TemplateToken token,
IDictionary<string, PipelineContextData> extraExpressionValues)
{
var result = default(DictionaryContextData);
if (token != null)
{
var templateContext = CreateTemplateContext(executionContext, extraExpressionValues);
try
{
token = TemplateEvaluator.Evaluate(templateContext, "outputs", token, 0, null, omitHeader: true);
templateContext.Errors.Check();
result = token.ToContextData().AssertDictionary("composite outputs");
}
catch (Exception ex) when (!(ex is TemplateValidationException))
{
templateContext.Errors.Add(ex);
}
templateContext.Errors.Check();
}
return result ?? new DictionaryContextData();
}
public List<string> EvaluateContainerArguments( public List<string> EvaluateContainerArguments(
IExecutionContext executionContext, IExecutionContext executionContext,
SequenceToken token, SequenceToken token,
@@ -139,11 +199,11 @@ namespace GitHub.Runner.Worker
if (token != null) if (token != null)
{ {
var context = CreateContext(executionContext, extraExpressionValues); var templateContext = CreateTemplateContext(executionContext, extraExpressionValues);
try try
{ {
var evaluateResult = TemplateEvaluator.Evaluate(context, "container-runs-args", token, 0, null, omitHeader: true); var evaluateResult = TemplateEvaluator.Evaluate(templateContext, "container-runs-args", token, 0, null, omitHeader: true);
context.Errors.Check(); templateContext.Errors.Check();
Trace.Info($"Arguments evaluate result: {StringUtil.ConvertToJson(evaluateResult)}"); Trace.Info($"Arguments evaluate result: {StringUtil.ConvertToJson(evaluateResult)}");
@@ -160,10 +220,10 @@ namespace GitHub.Runner.Worker
catch (Exception ex) when (!(ex is TemplateValidationException)) catch (Exception ex) when (!(ex is TemplateValidationException))
{ {
Trace.Error(ex); Trace.Error(ex);
context.Errors.Add(ex); templateContext.Errors.Add(ex);
} }
context.Errors.Check(); templateContext.Errors.Check();
} }
return result; return result;
@@ -178,11 +238,11 @@ namespace GitHub.Runner.Worker
if (token != null) if (token != null)
{ {
var context = CreateContext(executionContext, extraExpressionValues); var templateContext = CreateTemplateContext(executionContext, extraExpressionValues);
try try
{ {
var evaluateResult = TemplateEvaluator.Evaluate(context, "container-runs-env", token, 0, null, omitHeader: true); var evaluateResult = TemplateEvaluator.Evaluate(templateContext, "container-runs-env", token, 0, null, omitHeader: true);
context.Errors.Check(); templateContext.Errors.Check();
Trace.Info($"Environments evaluate result: {StringUtil.ConvertToJson(evaluateResult)}"); Trace.Info($"Environments evaluate result: {StringUtil.ConvertToJson(evaluateResult)}");
@@ -204,10 +264,10 @@ namespace GitHub.Runner.Worker
catch (Exception ex) when (!(ex is TemplateValidationException)) catch (Exception ex) when (!(ex is TemplateValidationException))
{ {
Trace.Error(ex); Trace.Error(ex);
context.Errors.Add(ex); templateContext.Errors.Add(ex);
} }
context.Errors.Check(); templateContext.Errors.Check();
} }
return result; return result;
@@ -221,11 +281,11 @@ namespace GitHub.Runner.Worker
string result = ""; string result = "";
if (token != null) if (token != null)
{ {
var context = CreateContext(executionContext); var templateContext = CreateTemplateContext(executionContext);
try try
{ {
var evaluateResult = TemplateEvaluator.Evaluate(context, "input-default-context", token, 0, null, omitHeader: true); var evaluateResult = TemplateEvaluator.Evaluate(templateContext, "input-default-context", token, 0, null, omitHeader: true);
context.Errors.Check(); templateContext.Errors.Check();
Trace.Info($"Input '{inputName}': default value evaluate result: {StringUtil.ConvertToJson(evaluateResult)}"); Trace.Info($"Input '{inputName}': default value evaluate result: {StringUtil.ConvertToJson(evaluateResult)}");
@@ -235,16 +295,16 @@ namespace GitHub.Runner.Worker
catch (Exception ex) when (!(ex is TemplateValidationException)) catch (Exception ex) when (!(ex is TemplateValidationException))
{ {
Trace.Error(ex); Trace.Error(ex);
context.Errors.Add(ex); templateContext.Errors.Add(ex);
} }
context.Errors.Check(); templateContext.Errors.Check();
} }
return result; return result;
} }
private TemplateContext CreateContext( private TemplateContext CreateTemplateContext(
IExecutionContext executionContext, IExecutionContext executionContext,
IDictionary<string, PipelineContextData> extraExpressionValues = null) IDictionary<string, PipelineContextData> extraExpressionValues = null)
{ {
@@ -281,21 +341,21 @@ namespace GitHub.Runner.Worker
result.ExpressionFunctions.Add(item); result.ExpressionFunctions.Add(item);
} }
// Add the file table // Add the file table from the Execution Context
if (_fileTable?.Count > 0) for (var i = 0; i < executionContext.Global.FileTable.Count; i++)
{ {
for (var i = 0 ; i < _fileTable.Count ; i++) result.GetFileId(executionContext.Global.FileTable[i]);
{
result.GetFileId(_fileTable[i]);
}
} }
return result; return result;
} }
private ActionExecutionData ConvertRuns( private ActionExecutionData ConvertRuns(
TemplateContext context, IExecutionContext executionContext,
TemplateToken inputsToken) TemplateContext templateContext,
TemplateToken inputsToken,
String fileRelativePath,
MappingToken outputs = null)
{ {
var runsMapping = inputsToken.AssertMapping("runs"); var runsMapping = inputsToken.AssertMapping("runs");
var usingToken = default(StringToken); var usingToken = default(StringToken);
@@ -311,6 +371,8 @@ namespace GitHub.Runner.Worker
var postToken = default(StringToken); var postToken = default(StringToken);
var postEntrypointToken = default(StringToken); var postEntrypointToken = default(StringToken);
var postIfToken = default(StringToken); var postIfToken = default(StringToken);
var steps = default(List<Pipelines.Step>);
foreach (var run in runsMapping) foreach (var run in runsMapping)
{ {
var runsKey = run.Key.AssertString("runs key").Value; var runsKey = run.Key.AssertString("runs key").Value;
@@ -355,6 +417,11 @@ namespace GitHub.Runner.Worker
case "pre-if": case "pre-if":
preIfToken = run.Value.AssertString("pre-if"); preIfToken = run.Value.AssertString("pre-if");
break; break;
case "steps":
var stepsToken = run.Value.AssertSequence("steps");
steps = PipelineTemplateConverter.ConvertToSteps(templateContext, stepsToken);
templateContext.Errors.Check();
break;
default: default:
Trace.Info($"Ignore run property {runsKey}."); Trace.Info($"Ignore run property {runsKey}.");
break; break;
@@ -367,7 +434,7 @@ namespace GitHub.Runner.Worker
{ {
if (string.IsNullOrEmpty(imageToken?.Value)) if (string.IsNullOrEmpty(imageToken?.Value))
{ {
throw new ArgumentNullException($"Image is not provided."); throw new ArgumentNullException($"You are using a Container Action but an image is not provided in {fileRelativePath}.");
} }
else else
{ {
@@ -388,7 +455,7 @@ namespace GitHub.Runner.Worker
{ {
if (string.IsNullOrEmpty(mainToken?.Value)) if (string.IsNullOrEmpty(mainToken?.Value))
{ {
throw new ArgumentNullException($"Entry javascript file is not provided."); throw new ArgumentNullException($"You are using a JavaScript Action but there is not an entry JavaScript file provided in {fileRelativePath}.");
} }
else else
{ {
@@ -402,6 +469,21 @@ namespace GitHub.Runner.Worker
}; };
} }
} }
else if (string.Equals(usingToken.Value, "composite", StringComparison.OrdinalIgnoreCase))
{
if (steps == null)
{
throw new ArgumentNullException($"You are using a composite action but there are no steps provided in {fileRelativePath}.");
}
else
{
return new CompositeActionExecutionData()
{
Steps = steps.Cast<Pipelines.ActionStep>().ToList(),
Outputs = outputs
};
}
}
else else
{ {
throw new ArgumentOutOfRangeException($"'using: {usingToken.Value}' is not supported, use 'docker' or 'node12' instead."); throw new ArgumentOutOfRangeException($"'using: {usingToken.Value}' is not supported, use 'docker' or 'node12' instead.");
@@ -419,7 +501,6 @@ namespace GitHub.Runner.Worker
} }
private void ConvertInputs( private void ConvertInputs(
TemplateContext context,
TemplateToken inputsToken, TemplateToken inputsToken,
ActionDefinitionData actionDefinition) ActionDefinitionData actionDefinition)
{ {

View File

@@ -94,6 +94,13 @@ namespace GitHub.Runner.Worker
if (handlerData.HasPost && (Stage == ActionRunStage.Pre || Stage == ActionRunStage.Main)) if (handlerData.HasPost && (Stage == ActionRunStage.Pre || Stage == ActionRunStage.Main))
{ {
string postDisplayName = $"Post {this.DisplayName}"; string postDisplayName = $"Post {this.DisplayName}";
if (Stage == ActionRunStage.Pre &&
this.DisplayName.StartsWith("Pre ", StringComparison.OrdinalIgnoreCase))
{
// Trim the leading `Pre ` from the display name.
// Otherwise, we will get `Post Pre xxx` as DisplayName for the Post step.
postDisplayName = $"Post {this.DisplayName.Substring("Pre ".Length)}";
}
var repositoryReference = Action.Reference as RepositoryPathReference; var repositoryReference = Action.Reference as RepositoryPathReference;
var pathString = string.IsNullOrEmpty(repositoryReference.Path) ? string.Empty : $"/{repositoryReference.Path}"; var pathString = string.IsNullOrEmpty(repositoryReference.Path) ? string.Empty : $"/{repositoryReference.Path}";
var repoString = string.IsNullOrEmpty(repositoryReference.Ref) ? $"{repositoryReference.Name}{pathString}" : var repoString = string.IsNullOrEmpty(repositoryReference.Ref) ? $"{repositoryReference.Name}{pathString}" :
@@ -128,16 +135,33 @@ namespace GitHub.Runner.Worker
ExecutionContext.SetGitHubContext("event_path", workflowFile); ExecutionContext.SetGitHubContext("event_path", workflowFile);
} }
// Set GITHUB_ACTION_REPOSITORY if this Action is from a repository
if (Action.Reference is Pipelines.RepositoryPathReference repoPathReferenceAction &&
!string.Equals(repoPathReferenceAction.RepositoryType, Pipelines.PipelineConstants.SelfAlias, StringComparison.OrdinalIgnoreCase))
{
ExecutionContext.SetGitHubContext("action_repository", repoPathReferenceAction.Name);
ExecutionContext.SetGitHubContext("action_ref", repoPathReferenceAction.Ref);
}
else
{
ExecutionContext.SetGitHubContext("action_repository", null);
ExecutionContext.SetGitHubContext("action_ref", null);
}
// Setup container stephost for running inside the container. // Setup container stephost for running inside the container.
if (ExecutionContext.Container != null) if (ExecutionContext.Global.Container != null)
{ {
// Make sure required container is already created. // Make sure required container is already created.
ArgUtil.NotNullOrEmpty(ExecutionContext.Container.ContainerId, nameof(ExecutionContext.Container.ContainerId)); ArgUtil.NotNullOrEmpty(ExecutionContext.Global.Container.ContainerId, nameof(ExecutionContext.Global.Container.ContainerId));
var containerStepHost = HostContext.CreateService<IContainerStepHost>(); var containerStepHost = HostContext.CreateService<IContainerStepHost>();
containerStepHost.Container = ExecutionContext.Container; containerStepHost.Container = ExecutionContext.Global.Container;
stepHost = containerStepHost; stepHost = containerStepHost;
} }
// Setup File Command Manager
var fileCommandManager = HostContext.CreateService<IFileCommandManager>();
fileCommandManager.InitializeFiles(ExecutionContext, null);
// Load the inputs. // Load the inputs.
ExecutionContext.Debug("Loading inputs"); ExecutionContext.Debug("Loading inputs");
var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator(); var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator();
@@ -155,6 +179,13 @@ namespace GitHub.Runner.Worker
} }
var validInputs = new HashSet<string>(StringComparer.OrdinalIgnoreCase); var validInputs = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (handlerData.ExecutionType == ActionExecutionType.Container)
{
// container action always accept 'entryPoint' and 'args' as inputs
// https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstepswithargs
validInputs.Add("entryPoint");
validInputs.Add("args");
}
// Merge the default inputs from the definition // Merge the default inputs from the definition
if (definition.Data?.Inputs != null) if (definition.Data?.Inputs != null)
{ {
@@ -170,11 +201,21 @@ namespace GitHub.Runner.Worker
} }
} }
foreach (var input in userInputs) // Validate inputs only for actions with action.yml
if (Action.Reference.Type == Pipelines.ActionSourceType.Repository)
{ {
if (!validInputs.Contains(input)) var unexpectedInputs = new List<string>();
foreach (var input in userInputs)
{ {
ExecutionContext.Warning($"Unexpected input '{input}', valid inputs are ['{string.Join("', '", validInputs)}']"); if (!validInputs.Contains(input))
{
unexpectedInputs.Add(input);
}
}
if (unexpectedInputs.Count > 0)
{
ExecutionContext.Warning($"Unexpected input(s) '{string.Join("', '", unexpectedInputs)}', valid inputs are ['{string.Join("', '", validInputs)}']");
} }
} }
@@ -207,14 +248,22 @@ namespace GitHub.Runner.Worker
handlerData, handlerData,
inputs, inputs,
environment, environment,
ExecutionContext.Variables, ExecutionContext.Global.Variables,
actionDirectory: definition.Directory); actionDirectory: definition.Directory);
// Print out action details // Print out action details
handler.PrintActionDetails(Stage); handler.PrintActionDetails(Stage);
// Run the task. // Run the task.
await handler.RunAsync(Stage); try
{
await handler.RunAsync(Stage);
}
finally
{
fileCommandManager.ProcessFiles(ExecutionContext, ExecutionContext.Global.Container);
}
} }
public bool TryEvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context) public bool TryEvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context)

View File

@@ -21,6 +21,11 @@ namespace GitHub.Runner.Worker.Container
{ {
} }
public ContainerInfo(IHostContext hostContext)
{
UpdateWebProxyEnv(hostContext.WebProxy);
}
public ContainerInfo(IHostContext hostContext, Pipelines.JobContainer container, bool isJobContainer = true, string networkAlias = null) public ContainerInfo(IHostContext hostContext, Pipelines.JobContainer container, bool isJobContainer = true, string networkAlias = null)
{ {
this.ContainerName = container.Alias; this.ContainerName = container.Alias;
@@ -34,6 +39,9 @@ namespace GitHub.Runner.Worker.Container
_environmentVariables = container.Environment; _environmentVariables = container.Environment;
this.IsJobContainer = isJobContainer; this.IsJobContainer = isJobContainer;
this.ContainerNetworkAlias = networkAlias; this.ContainerNetworkAlias = networkAlias;
this.RegistryAuthUsername = container.Credentials?.Username;
this.RegistryAuthPassword = container.Credentials?.Password;
this.RegistryServer = DockerUtil.ParseRegistryHostnameFromImageName(this.ContainerImage);
#if OS_WINDOWS #if OS_WINDOWS
_pathMappings.Add(new PathMapping(hostContext.GetDirectory(WellKnownDirectory.Work), "C:\\__w")); _pathMappings.Add(new PathMapping(hostContext.GetDirectory(WellKnownDirectory.Work), "C:\\__w"));
@@ -79,6 +87,9 @@ namespace GitHub.Runner.Worker.Container
public string ContainerWorkDirectory { get; set; } public string ContainerWorkDirectory { get; set; }
public string ContainerCreateOptions { get; private set; } public string ContainerCreateOptions { get; private set; }
public string ContainerRuntimePath { get; set; } public string ContainerRuntimePath { get; set; }
public string RegistryServer { get; set; }
public string RegistryAuthUsername { get; set; }
public string RegistryAuthPassword { get; set; }
public bool IsJobContainer { get; set; } public bool IsJobContainer { get; set; }
public IDictionary<string, string> ContainerEnvironmentVariables public IDictionary<string, string> ContainerEnvironmentVariables

View File

@@ -4,6 +4,7 @@ using System.IO;
using System.Linq; using System.Linq;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading; using System.Threading;
using System.Threading.Channels;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.Runner.Common; using GitHub.Runner.Common;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;
@@ -17,6 +18,7 @@ namespace GitHub.Runner.Worker.Container
string DockerInstanceLabel { get; } string DockerInstanceLabel { get; }
Task<DockerVersion> DockerVersion(IExecutionContext context); Task<DockerVersion> DockerVersion(IExecutionContext context);
Task<int> DockerPull(IExecutionContext context, string image); Task<int> DockerPull(IExecutionContext context, string image);
Task<int> DockerPull(IExecutionContext context, string image, string configFileDirectory);
Task<int> DockerBuild(IExecutionContext context, string workingDirectory, string dockerFile, string dockerContext, string tag); Task<int> DockerBuild(IExecutionContext context, string workingDirectory, string dockerFile, string dockerContext, string tag);
Task<string> DockerCreate(IExecutionContext context, ContainerInfo container); Task<string> DockerCreate(IExecutionContext context, ContainerInfo container);
Task<int> DockerRun(IExecutionContext context, ContainerInfo container, EventHandler<ProcessDataReceivedEventArgs> stdoutDataReceived, EventHandler<ProcessDataReceivedEventArgs> stderrDataReceived); Task<int> DockerRun(IExecutionContext context, ContainerInfo container, EventHandler<ProcessDataReceivedEventArgs> stdoutDataReceived, EventHandler<ProcessDataReceivedEventArgs> stderrDataReceived);
@@ -31,6 +33,7 @@ namespace GitHub.Runner.Worker.Container
Task<int> DockerExec(IExecutionContext context, string containerId, string options, string command, List<string> outputs); Task<int> DockerExec(IExecutionContext context, string containerId, string options, string command, List<string> outputs);
Task<List<string>> DockerInspect(IExecutionContext context, string dockerObject, string options); Task<List<string>> DockerInspect(IExecutionContext context, string dockerObject, string options);
Task<List<PortMapping>> DockerPort(IExecutionContext context, string containerId); Task<List<PortMapping>> DockerPort(IExecutionContext context, string containerId);
Task<int> DockerLogin(IExecutionContext context, string configFileDirectory, string registry, string username, string password);
} }
public class DockerCommandManager : RunnerService, IDockerCommandManager public class DockerCommandManager : RunnerService, IDockerCommandManager
@@ -82,9 +85,18 @@ namespace GitHub.Runner.Worker.Container
return new DockerVersion(serverVersion, clientVersion); return new DockerVersion(serverVersion, clientVersion);
} }
public async Task<int> DockerPull(IExecutionContext context, string image) public Task<int> DockerPull(IExecutionContext context, string image)
{ {
return await ExecuteDockerCommandAsync(context, "pull", image, context.CancellationToken); return DockerPull(context, image, null);
}
public async Task<int> DockerPull(IExecutionContext context, string image, string configFileDirectory)
{
if (string.IsNullOrEmpty(configFileDirectory))
{
return await ExecuteDockerCommandAsync(context, $"pull", image, context.CancellationToken);
}
return await ExecuteDockerCommandAsync(context, $"--config {configFileDirectory} pull", image, context.CancellationToken);
} }
public async Task<int> DockerBuild(IExecutionContext context, string workingDirectory, string dockerFile, string dockerContext, string tag) public async Task<int> DockerBuild(IExecutionContext context, string workingDirectory, string dockerFile, string dockerContext, string tag)
@@ -346,6 +358,28 @@ namespace GitHub.Runner.Worker.Container
return DockerUtil.ParseDockerPort(portMappingLines); return DockerUtil.ParseDockerPort(portMappingLines);
} }
public Task<int> DockerLogin(IExecutionContext context, string configFileDirectory, string registry, string username, string password)
{
string args = $"--config {configFileDirectory} login {registry} -u {username} --password-stdin";
context.Command($"{DockerPath} {args}");
var input = Channel.CreateBounded<string>(new BoundedChannelOptions(1) { SingleReader = true, SingleWriter = true });
input.Writer.TryWrite(password);
var processInvoker = HostContext.CreateService<IProcessInvoker>();
return processInvoker.ExecuteAsync(
workingDirectory: context.GetGitHubContext("workspace"),
fileName: DockerPath,
arguments: args,
environment: null,
requireExitCodeZero: false,
outputEncoding: null,
killProcessOnCancel: false,
redirectStandardIn: input,
cancellationToken: context.CancellationToken);
}
private Task<int> ExecuteDockerCommandAsync(IExecutionContext context, string command, string options, CancellationToken cancellationToken = default(CancellationToken)) private Task<int> ExecuteDockerCommandAsync(IExecutionContext context, string command, string options, CancellationToken cancellationToken = default(CancellationToken))
{ {
return ExecuteDockerCommandAsync(context, command, options, null, cancellationToken); return ExecuteDockerCommandAsync(context, command, options, null, cancellationToken);

View File

@@ -45,5 +45,21 @@ namespace GitHub.Runner.Worker.Container
} }
return ""; return "";
} }
public static string ParseRegistryHostnameFromImageName(string name)
{
var nameSplit = name.Split('/');
// Single slash is implictly from Dockerhub, unless first part has .tld or :port
if (nameSplit.Length == 2 && (nameSplit[0].Contains(":") || nameSplit[0].Contains(".")))
{
return nameSplit[0];
}
// All other non Dockerhub registries
else if (nameSplit.Length > 2)
{
return nameSplit[0];
}
return "";
}
} }
} }

View File

@@ -91,7 +91,10 @@ namespace GitHub.Runner.Worker
#endif #endif
// Check docker client/server version // Check docker client/server version
executionContext.Output("##[group]Checking docker version");
DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext); DockerVersion dockerVersion = await _dockerManger.DockerVersion(executionContext);
executionContext.Output("##[endgroup]");
ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion)); ArgUtil.NotNull(dockerVersion.ServerVersion, nameof(dockerVersion.ServerVersion));
ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion)); ArgUtil.NotNull(dockerVersion.ClientVersion, nameof(dockerVersion.ClientVersion));
@@ -111,7 +114,7 @@ namespace GitHub.Runner.Worker
} }
// Clean up containers left by previous runs // Clean up containers left by previous runs
executionContext.Debug($"Delete stale containers from previous jobs"); executionContext.Output("##[group]Clean up resources from previous jobs");
var staleContainers = await _dockerManger.DockerPS(executionContext, $"--all --quiet --no-trunc --filter \"label={_dockerManger.DockerInstanceLabel}\""); var staleContainers = await _dockerManger.DockerPS(executionContext, $"--all --quiet --no-trunc --filter \"label={_dockerManger.DockerInstanceLabel}\"");
foreach (var staleContainer in staleContainers) foreach (var staleContainer in staleContainers)
{ {
@@ -122,18 +125,20 @@ namespace GitHub.Runner.Worker
} }
} }
executionContext.Debug($"Delete stale container networks from previous jobs");
int networkPruneExitCode = await _dockerManger.DockerNetworkPrune(executionContext); int networkPruneExitCode = await _dockerManger.DockerNetworkPrune(executionContext);
if (networkPruneExitCode != 0) if (networkPruneExitCode != 0)
{ {
executionContext.Warning($"Delete stale container networks failed, docker network prune fail with exit code {networkPruneExitCode}"); executionContext.Warning($"Delete stale container networks failed, docker network prune fail with exit code {networkPruneExitCode}");
} }
executionContext.Output("##[endgroup]");
// Create local docker network for this job to avoid port conflict when multiple runners run on same machine. // Create local docker network for this job to avoid port conflict when multiple runners run on same machine.
// All containers within a job join the same network // All containers within a job join the same network
executionContext.Output("##[group]Create local container network");
var containerNetwork = $"github_network_{Guid.NewGuid().ToString("N")}"; var containerNetwork = $"github_network_{Guid.NewGuid().ToString("N")}";
await CreateContainerNetworkAsync(executionContext, containerNetwork); await CreateContainerNetworkAsync(executionContext, containerNetwork);
executionContext.JobContext.Container["network"] = new StringContextData(containerNetwork); executionContext.JobContext.Container["network"] = new StringContextData(containerNetwork);
executionContext.Output("##[endgroup]");
foreach (var container in containers) foreach (var container in containers)
{ {
@@ -141,10 +146,12 @@ namespace GitHub.Runner.Worker
await StartContainerAsync(executionContext, container); await StartContainerAsync(executionContext, container);
} }
executionContext.Output("##[group]Waiting for all services to be ready");
foreach (var container in containers.Where(c => !c.IsJobContainer)) foreach (var container in containers.Where(c => !c.IsJobContainer))
{ {
await ContainerHealthcheck(executionContext, container); await ContainerHealthcheck(executionContext, container);
} }
executionContext.Output("##[endgroup]");
} }
public async Task StopContainersAsync(IExecutionContext executionContext, object data) public async Task StopContainersAsync(IExecutionContext executionContext, object data)
@@ -173,6 +180,10 @@ namespace GitHub.Runner.Worker
Trace.Info($"Container name: {container.ContainerName}"); Trace.Info($"Container name: {container.ContainerName}");
Trace.Info($"Container image: {container.ContainerImage}"); Trace.Info($"Container image: {container.ContainerImage}");
Trace.Info($"Container options: {container.ContainerCreateOptions}"); Trace.Info($"Container options: {container.ContainerCreateOptions}");
var groupName = container.IsJobContainer ? "Starting job container" : $"Starting {container.ContainerNetworkAlias} service container";
executionContext.Output($"##[group]{groupName}");
foreach (var port in container.UserPortMappings) foreach (var port in container.UserPortMappings)
{ {
Trace.Info($"User provided port: {port.Value}"); Trace.Info($"User provided port: {port.Value}");
@@ -187,12 +198,18 @@ namespace GitHub.Runner.Worker
} }
} }
// TODO: Add at a later date. This currently no local package registry to test with
// UpdateRegistryAuthForGitHubToken(executionContext, container);
// Before pulling, generate client authentication if required
var configLocation = await ContainerRegistryLogin(executionContext, container);
// Pull down docker image with retry up to 3 times // Pull down docker image with retry up to 3 times
int retryCount = 0; int retryCount = 0;
int pullExitCode = 0; int pullExitCode = 0;
while (retryCount < 3) while (retryCount < 3)
{ {
pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage); pullExitCode = await _dockerManger.DockerPull(executionContext, container.ContainerImage, configLocation);
if (pullExitCode == 0) if (pullExitCode == 0)
{ {
break; break;
@@ -209,6 +226,9 @@ namespace GitHub.Runner.Worker
} }
} }
// Remove credentials after pulling
ContainerRegistryLogout(configLocation);
if (retryCount == 3 && pullExitCode != 0) if (retryCount == 3 && pullExitCode != 0)
{ {
throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}"); throw new InvalidOperationException($"Docker pull failed with exit code {pullExitCode}");
@@ -304,6 +324,7 @@ namespace GitHub.Runner.Worker
container.ContainerRuntimePath = DockerUtil.ParsePathFromConfigEnv(containerEnv); container.ContainerRuntimePath = DockerUtil.ParsePathFromConfigEnv(containerEnv);
executionContext.JobContext.Container["id"] = new StringContextData(container.ContainerId); executionContext.JobContext.Container["id"] = new StringContextData(container.ContainerId);
} }
executionContext.Output("##[endgroup]");
} }
private async Task StopContainerAsync(IExecutionContext executionContext, ContainerInfo container) private async Task StopContainerAsync(IExecutionContext executionContext, ContainerInfo container)
@@ -425,5 +446,83 @@ namespace GitHub.Runner.Worker
throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}."); throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}.");
} }
} }
private async Task<string> ContainerRegistryLogin(IExecutionContext executionContext, ContainerInfo container)
{
if (string.IsNullOrEmpty(container.RegistryAuthUsername) || string.IsNullOrEmpty(container.RegistryAuthPassword))
{
// No valid client config can be generated
return "";
}
var configLocation = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Temp), $".docker_{Guid.NewGuid()}");
try
{
var dirInfo = Directory.CreateDirectory(configLocation);
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to create directory to store registry client credentials: {e.Message}");
}
var loginExitCode = await _dockerManger.DockerLogin(
executionContext,
configLocation,
container.RegistryServer,
container.RegistryAuthUsername,
container.RegistryAuthPassword);
if (loginExitCode != 0)
{
throw new InvalidOperationException($"Docker login for '{container.RegistryServer}' failed with exit code {loginExitCode}");
}
return configLocation;
}
private void ContainerRegistryLogout(string configLocation)
{
try
{
if (!string.IsNullOrEmpty(configLocation) && Directory.Exists(configLocation))
{
Directory.Delete(configLocation, recursive: true);
}
}
catch (Exception e)
{
throw new InvalidOperationException($"Failed to remove directory containing Docker client credentials: {e.Message}");
}
}
private void UpdateRegistryAuthForGitHubToken(IExecutionContext executionContext, ContainerInfo container)
{
var registryIsTokenCompatible = container.RegistryServer.Equals("docker.pkg.github.com", StringComparison.OrdinalIgnoreCase);
if (!registryIsTokenCompatible)
{
return;
}
var registryMatchesWorkflow = false;
// REGISTRY/OWNER/REPO/IMAGE[:TAG]
var imageParts = container.ContainerImage.Split('/');
if (imageParts.Length != 4)
{
executionContext.Warning($"Could not identify owner and repo for container image {container.ContainerImage}. Skipping automatic token auth");
return;
}
var owner = imageParts[1];
var repo = imageParts[2];
var nwo = $"{owner}/{repo}";
if (nwo.Equals(executionContext.GetGitHubContext("repository"), StringComparison.OrdinalIgnoreCase))
{
registryMatchesWorkflow = true;
}
var registryCredentialsNotSupplied = string.IsNullOrEmpty(container.RegistryAuthUsername) && string.IsNullOrEmpty(container.RegistryAuthPassword);
if (registryCredentialsNotSupplied && registryMatchesWorkflow)
{
container.RegistryAuthUsername = executionContext.GetGitHubContext("actor");
container.RegistryAuthPassword = executionContext.GetGitHubContext("token");
}
}
} }
} }

View File

@@ -86,9 +86,9 @@ namespace GitHub.Runner.Worker
executionContext.Debug("Zipping diagnostic files."); executionContext.Debug("Zipping diagnostic files.");
string buildNumber = executionContext.Variables.Build_Number ?? "UnknownBuildNumber"; string buildNumber = executionContext.Global.Variables.Build_Number ?? "UnknownBuildNumber";
string buildName = $"Build {buildNumber}"; string buildName = $"Build {buildNumber}";
string phaseName = executionContext.Variables.System_PhaseDisplayName ?? "UnknownPhaseName"; string phaseName = executionContext.Global.Variables.System_PhaseDisplayName ?? "UnknownPhaseName";
// zip the files // zip the files
string diagnosticsZipFileName = $"{buildName}-{phaseName}.zip"; string diagnosticsZipFileName = $"{buildName}-{phaseName}.zip";

View File

@@ -6,6 +6,7 @@ using System.Globalization;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using System.Text.RegularExpressions;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Web; using System.Web;
@@ -43,22 +44,13 @@ namespace GitHub.Runner.Worker
string ResultCode { get; set; } string ResultCode { get; set; }
TaskResult? CommandResult { get; set; } TaskResult? CommandResult { get; set; }
CancellationToken CancellationToken { get; } CancellationToken CancellationToken { get; }
List<ServiceEndpoint> Endpoints { get; } GlobalContext Global { get; }
PlanFeatures Features { get; }
Variables Variables { get; }
Dictionary<string, string> IntraActionState { get; } Dictionary<string, string> IntraActionState { get; }
IDictionary<String, IDictionary<String, String>> JobDefaults { get; }
Dictionary<string, VariableValue> JobOutputs { get; } Dictionary<string, VariableValue> JobOutputs { get; }
IDictionary<String, String> EnvironmentVariables { get; } ActionsEnvironmentReference ActionsEnvironment { get; }
IDictionary<String, ContextScope> Scopes { get; }
IList<String> FileTable { get; }
StepsContext StepsContext { get; }
DictionaryContextData ExpressionValues { get; } DictionaryContextData ExpressionValues { get; }
IList<IFunctionInfo> ExpressionFunctions { get; } IList<IFunctionInfo> ExpressionFunctions { get; }
List<string> PrependPath { get; }
ContainerInfo Container { get; set; }
List<ContainerInfo> ServiceContainers { get; }
JobContext JobContext { get; } JobContext JobContext { get; }
// Only job level ExecutionContext has JobSteps // Only job level ExecutionContext has JobSteps
@@ -69,13 +61,16 @@ namespace GitHub.Runner.Worker
bool EchoOnActionCommand { get; set; } bool EchoOnActionCommand { get; set; }
bool InsideComposite { get; }
ExecutionContext Root { get; }
// Initialize // Initialize
void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token); void InitializeJob(Pipelines.AgentJobRequestMessage message, CancellationToken token);
void CancelToken(); void CancelToken();
IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary<string, string> intraActionState = null, int? recordOrder = null); IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool insideComposite = false, CancellationTokenSource cancellationTokenSource = null);
// logging // logging
bool WriteDebug { get; }
long Write(string tag, string message); long Write(string tag, string message);
void QueueAttachFile(string type, string name, string filePath); void QueueAttachFile(string type, string name, string filePath);
@@ -104,11 +99,13 @@ namespace GitHub.Runner.Worker
// others // others
void ForceTaskComplete(); void ForceTaskComplete();
void RegisterPostJobStep(IStep step); void RegisterPostJobStep(IStep step);
IStep CreateCompositeStep(string scopeName, IActionRunner step, DictionaryContextData inputsData, Dictionary<string, string> envData);
} }
public sealed class ExecutionContext : RunnerService, IExecutionContext public sealed class ExecutionContext : RunnerService, IExecutionContext
{ {
private const int _maxIssueCount = 10; private const int _maxIssueCount = 10;
private const int _throttlingDelayReportThreshold = 10 * 1000; // Don't report throttling with less than 10 seconds delay
private readonly TimelineRecord _record = new TimelineRecord(); private readonly TimelineRecord _record = new TimelineRecord();
private readonly Dictionary<Guid, TimelineRecord> _detailRecords = new Dictionary<Guid, TimelineRecord>(); private readonly Dictionary<Guid, TimelineRecord> _detailRecords = new Dictionary<Guid, TimelineRecord>();
@@ -139,21 +136,15 @@ namespace GitHub.Runner.Worker
public string ContextName { get; private set; } public string ContextName { get; private set; }
public Task ForceCompleted => _forceCompleted.Task; public Task ForceCompleted => _forceCompleted.Task;
public CancellationToken CancellationToken => _cancellationTokenSource.Token; public CancellationToken CancellationToken => _cancellationTokenSource.Token;
public List<ServiceEndpoint> Endpoints { get; private set; }
public Variables Variables { get; private set; }
public Dictionary<string, string> IntraActionState { get; private set; } public Dictionary<string, string> IntraActionState { get; private set; }
public IDictionary<String, IDictionary<String, String>> JobDefaults { get; private set; }
public Dictionary<string, VariableValue> JobOutputs { get; private set; } public Dictionary<string, VariableValue> JobOutputs { get; private set; }
public IDictionary<String, String> EnvironmentVariables { get; private set; }
public IDictionary<String, ContextScope> Scopes { get; private set; } public ActionsEnvironmentReference ActionsEnvironment { get; private set; }
public IList<String> FileTable { get; private set; }
public StepsContext StepsContext { get; private set; }
public DictionaryContextData ExpressionValues { get; } = new DictionaryContextData(); public DictionaryContextData ExpressionValues { get; } = new DictionaryContextData();
public IList<IFunctionInfo> ExpressionFunctions { get; } = new List<IFunctionInfo>(); public IList<IFunctionInfo> ExpressionFunctions { get; } = new List<IFunctionInfo>();
public bool WriteDebug { get; private set; }
public List<string> PrependPath { get; private set; } // Shared pointer across job-level execution context and step-level execution contexts
public ContainerInfo Container { get; set; } public GlobalContext Global { get; private set; }
public List<ContainerInfo> ServiceContainers { get; private set; }
// Only job level ExecutionContext has JobSteps // Only job level ExecutionContext has JobSteps
public Queue<IStep> JobSteps { get; private set; } public Queue<IStep> JobSteps { get; private set; }
@@ -166,6 +157,7 @@ namespace GitHub.Runner.Worker
public bool EchoOnActionCommand { get; set; } public bool EchoOnActionCommand { get; set; }
public bool InsideComposite { get; private set; }
public TaskResult? Result public TaskResult? Result
{ {
@@ -197,9 +189,7 @@ namespace GitHub.Runner.Worker
} }
} }
public PlanFeatures Features { get; private set; } public ExecutionContext Root
private ExecutionContext Root
{ {
get get
{ {
@@ -263,17 +253,44 @@ namespace GitHub.Runner.Worker
Root.PostJobSteps.Push(step); Root.PostJobSteps.Push(step);
} }
public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary<string, string> intraActionState = null, int? recordOrder = null) /// <summary>
/// Helper function used in CompositeActionHandler::RunAsync to
/// add a child node, aka a step, to the current job to the Root.JobSteps based on the location.
/// </summary>
public IStep CreateCompositeStep(
string scopeName,
IActionRunner step,
DictionaryContextData inputsData,
Dictionary<string, string> envData)
{
step.ExecutionContext = Root.CreateChild(_record.Id, _record.Name, _record.Id.ToString("N"), scopeName, step.Action.ContextName, logger: _logger, insideComposite: true, cancellationTokenSource: CancellationTokenSource.CreateLinkedTokenSource(_cancellationTokenSource.Token));
step.ExecutionContext.ExpressionValues["inputs"] = inputsData;
step.ExecutionContext.ExpressionValues["steps"] = Global.StepsContext.GetScope(step.ExecutionContext.GetFullyQualifiedContextName());
// Add the composite action environment variables to each step.
#if OS_WINDOWS
var envContext = new DictionaryContextData();
#else
var envContext = new CaseSensitiveDictionaryContextData();
#endif
foreach (var pair in envData)
{
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
step.ExecutionContext.ExpressionValues["env"] = envContext;
return step;
}
public IExecutionContext CreateChild(Guid recordId, string displayName, string refName, string scopeName, string contextName, Dictionary<string, string> intraActionState = null, int? recordOrder = null, IPagingLogger logger = null, bool insideComposite = false, CancellationTokenSource cancellationTokenSource = null)
{ {
Trace.Entering(); Trace.Entering();
var child = new ExecutionContext(); var child = new ExecutionContext();
child.Initialize(HostContext); child.Initialize(HostContext);
child.Global = Global;
child.ScopeName = scopeName; child.ScopeName = scopeName;
child.ContextName = contextName; child.ContextName = contextName;
child.Features = Features;
child.Variables = Variables;
child.Endpoints = Endpoints;
if (intraActionState == null) if (intraActionState == null)
{ {
child.IntraActionState = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); child.IntraActionState = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
@@ -282,11 +299,6 @@ namespace GitHub.Runner.Worker
{ {
child.IntraActionState = intraActionState; child.IntraActionState = intraActionState;
} }
child.EnvironmentVariables = EnvironmentVariables;
child.JobDefaults = JobDefaults;
child.Scopes = Scopes;
child.FileTable = FileTable;
child.StepsContext = StepsContext;
foreach (var pair in ExpressionValues) foreach (var pair in ExpressionValues)
{ {
child.ExpressionValues[pair.Key] = pair.Value; child.ExpressionValues[pair.Key] = pair.Value;
@@ -295,12 +307,8 @@ namespace GitHub.Runner.Worker
{ {
child.ExpressionFunctions.Add(item); child.ExpressionFunctions.Add(item);
} }
child._cancellationTokenSource = new CancellationTokenSource(); child._cancellationTokenSource = cancellationTokenSource ?? new CancellationTokenSource();
child.WriteDebug = WriteDebug;
child._parentExecutionContext = this; child._parentExecutionContext = this;
child.PrependPath = PrependPath;
child.Container = Container;
child.ServiceContainers = ServiceContainers;
child.EchoOnActionCommand = EchoOnActionCommand; child.EchoOnActionCommand = EchoOnActionCommand;
if (recordOrder != null) if (recordOrder != null)
@@ -311,9 +319,17 @@ namespace GitHub.Runner.Worker
{ {
child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, ++_childTimelineRecordOrder); child.InitializeTimelineRecord(_mainTimelineId, recordId, _record.Id, ExecutionContextType.Task, displayName, refName, ++_childTimelineRecordOrder);
} }
if (logger != null)
{
child._logger = logger;
}
else
{
child._logger = HostContext.CreateService<IPagingLogger>();
child._logger.Setup(_mainTimelineId, recordId);
}
child._logger = HostContext.CreateService<IPagingLogger>(); child.InsideComposite = insideComposite;
child._logger.Setup(_mainTimelineId, recordId);
return child; return child;
} }
@@ -335,7 +351,7 @@ namespace GitHub.Runner.Worker
} }
// report total delay caused by server throttling. // report total delay caused by server throttling.
if (_totalThrottlingDelayInMilliseconds > 0) if (_totalThrottlingDelayInMilliseconds > _throttlingDelayReportThreshold)
{ {
this.Warning($"The job has experienced {TimeSpan.FromMilliseconds(_totalThrottlingDelayInMilliseconds).TotalSeconds} seconds total delay caused by server throttling."); this.Warning($"The job has experienced {TimeSpan.FromMilliseconds(_totalThrottlingDelayInMilliseconds).TotalSeconds} seconds total delay caused by server throttling.");
} }
@@ -363,14 +379,19 @@ namespace GitHub.Runner.Worker
} }
} }
_cancellationTokenSource?.Dispose(); if (Root != this)
{
// only dispose TokenSource for step level ExecutionContext
_cancellationTokenSource?.Dispose();
}
_logger.End(); _logger.End();
if (!string.IsNullOrEmpty(ContextName)) // Skip if generated context name. Generated context names start with "__". After M271-ish the server will never send an empty context name.
if (!string.IsNullOrEmpty(ContextName) && !ContextName.StartsWith("__", StringComparison.Ordinal))
{ {
StepsContext.SetOutcome(ScopeName, ContextName, (Outcome ?? Result ?? TaskResult.Succeeded).ToActionResult()); Global.StepsContext.SetOutcome(ScopeName, ContextName, (Outcome ?? Result ?? TaskResult.Succeeded).ToActionResult());
StepsContext.SetConclusion(ScopeName, ContextName, (Result ?? TaskResult.Succeeded).ToActionResult()); Global.StepsContext.SetConclusion(ScopeName, ContextName, (Result ?? TaskResult.Succeeded).ToActionResult());
} }
return Result.Value; return Result.Value;
@@ -429,7 +450,8 @@ namespace GitHub.Runner.Worker
{ {
ArgUtil.NotNullOrEmpty(name, nameof(name)); ArgUtil.NotNullOrEmpty(name, nameof(name));
if (String.IsNullOrEmpty(ContextName)) // Skip if generated context name. Generated context names start with "__". After M271-ish the server will never send an empty context name.
if (string.IsNullOrEmpty(ContextName) || ContextName.StartsWith("__", StringComparison.Ordinal))
{ {
reference = null; reference = null;
return; return;
@@ -437,7 +459,7 @@ namespace GitHub.Runner.Worker
// todo: restrict multiline? // todo: restrict multiline?
StepsContext.SetOutput(ScopeName, ContextName, name, value, out reference); Global.StepsContext.SetOutput(ScopeName, ContextName, name, value, out reference);
} }
public void SetTimeout(TimeSpan? timeout) public void SetTimeout(TimeSpan? timeout)
@@ -571,42 +593,38 @@ namespace GitHub.Runner.Worker
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); _cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token);
// Features Global = new GlobalContext();
Features = PlanUtil.GetFeatures(message.Plan);
// Plan
Global.Plan = message.Plan;
Global.Features = PlanUtil.GetFeatures(message.Plan);
// Endpoints // Endpoints
Endpoints = message.Resources.Endpoints; Global.Endpoints = message.Resources.Endpoints;
// Variables // Variables
Variables = new Variables(HostContext, message.Variables); Global.Variables = new Variables(HostContext, message.Variables);
// Environment variables shared across all actions // Environment variables shared across all actions
EnvironmentVariables = new Dictionary<string, string>(VarUtil.EnvironmentVariableKeyComparer); Global.EnvironmentVariables = new Dictionary<string, string>(VarUtil.EnvironmentVariableKeyComparer);
// Job defaults shared across all actions // Job defaults shared across all actions
JobDefaults = new Dictionary<string, IDictionary<string, string>>(StringComparer.OrdinalIgnoreCase); Global.JobDefaults = new Dictionary<string, IDictionary<string, string>>(StringComparer.OrdinalIgnoreCase);
// Job Outputs // Job Outputs
JobOutputs = new Dictionary<string, VariableValue>(StringComparer.OrdinalIgnoreCase); JobOutputs = new Dictionary<string, VariableValue>(StringComparer.OrdinalIgnoreCase);
// Actions environment
ActionsEnvironment = message.ActionsEnvironment;
// Service container info // Service container info
ServiceContainers = new List<ContainerInfo>(); Global.ServiceContainers = new List<ContainerInfo>();
// Steps context (StepsRunner manages adding the scoped steps context) // Steps context (StepsRunner manages adding the scoped steps context)
StepsContext = new StepsContext(); Global.StepsContext = new StepsContext();
// Scopes
Scopes = new Dictionary<String, ContextScope>(StringComparer.OrdinalIgnoreCase);
if (message.Scopes?.Count > 0)
{
foreach (var scope in message.Scopes)
{
Scopes[scope.Name] = scope;
}
}
// File table // File table
FileTable = new List<String>(message.FileTable ?? new string[0]); Global.FileTable = new List<String>(message.FileTable ?? new string[0]);
// Expression values // Expression values
if (message.ContextData?.Count > 0) if (message.ContextData?.Count > 0)
@@ -617,15 +635,15 @@ namespace GitHub.Runner.Worker
} }
} }
ExpressionValues["secrets"] = Variables.ToSecretsContext(); ExpressionValues["secrets"] = Global.Variables.ToSecretsContext();
ExpressionValues["runner"] = new RunnerContext(); ExpressionValues["runner"] = new RunnerContext();
ExpressionValues["job"] = new JobContext(); ExpressionValues["job"] = new JobContext();
Trace.Info("Initialize GitHub context"); Trace.Info("Initialize GitHub context");
var githubAccessToken = new StringContextData(Variables.Get("system.github.token")); var githubAccessToken = new StringContextData(Global.Variables.Get("system.github.token"));
var base64EncodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{githubAccessToken}")); var base64EncodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes($"x-access-token:{githubAccessToken}"));
HostContext.SecretMasker.AddValue(base64EncodedToken); HostContext.SecretMasker.AddValue(base64EncodedToken);
var githubJob = Variables.Get("system.github.job"); var githubJob = Global.Variables.Get("system.github.job");
var githubContext = new GitHubContext(); var githubContext = new GitHubContext();
githubContext["token"] = githubAccessToken; githubContext["token"] = githubAccessToken;
if (!string.IsNullOrEmpty(githubJob)) if (!string.IsNullOrEmpty(githubJob))
@@ -648,7 +666,7 @@ namespace GitHub.Runner.Worker
#endif #endif
// Prepend Path // Prepend Path
PrependPath = new List<string>(); Global.PrependPath = new List<string>();
// JobSteps for job ExecutionContext // JobSteps for job ExecutionContext
JobSteps = new Queue<IStep>(); JobSteps = new Queue<IStep>();
@@ -674,10 +692,10 @@ namespace GitHub.Runner.Worker
_logger.Setup(_mainTimelineId, _record.Id); _logger.Setup(_mainTimelineId, _record.Id);
// Initialize 'echo on action command success' property, default to false, unless Step_Debug is set // Initialize 'echo on action command success' property, default to false, unless Step_Debug is set
EchoOnActionCommand = Variables.Step_Debug ?? false; EchoOnActionCommand = Global.Variables.Step_Debug ?? false;
// Verbosity (from GitHub.Step_Debug). // Verbosity (from GitHub.Step_Debug).
WriteDebug = Variables.Step_Debug ?? false; Global.WriteDebug = Global.Variables.Step_Debug ?? false;
// Hook up JobServerQueueThrottling event, we will log warning on server tarpit. // Hook up JobServerQueueThrottling event, we will log warning on server tarpit.
_jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived; _jobServerQueue.JobServerQueueThrottling += JobServerQueueThrottling_EventReceived;
@@ -705,7 +723,7 @@ namespace GitHub.Runner.Worker
} }
} }
_jobServerQueue.QueueWebConsoleLine(_record.Id, msg); _jobServerQueue.QueueWebConsoleLine(_record.Id, msg, totalLines);
return totalLines; return totalLines;
} }
@@ -851,7 +869,8 @@ namespace GitHub.Runner.Worker
{ {
Interlocked.Add(ref _totalThrottlingDelayInMilliseconds, Convert.ToInt64(data.Delay.TotalMilliseconds)); Interlocked.Add(ref _totalThrottlingDelayInMilliseconds, Convert.ToInt64(data.Delay.TotalMilliseconds));
if (!_throttlingReported) if (!_throttlingReported &&
_totalThrottlingDelayInMilliseconds > _throttlingDelayReportThreshold)
{ {
this.Warning(string.Format("The job is currently being throttled by the server. You may experience delays in console line output, job status reporting, and action log uploads.")); this.Warning(string.Format("The job is currently being throttled by the server. You may experience delays in console line output, job status reporting, and action log uploads."));
@@ -877,6 +896,16 @@ namespace GitHub.Runner.Worker
// Otherwise individual overloads would need to be implemented (depending on the unit test). // Otherwise individual overloads would need to be implemented (depending on the unit test).
public static class ExecutionContextExtension public static class ExecutionContextExtension
{ {
public static string GetFullyQualifiedContextName(this IExecutionContext context)
{
if (!string.IsNullOrEmpty(context.ScopeName))
{
return $"{context.ScopeName}.{context.ContextName}";
}
return context.ContextName;
}
public static void Error(this IExecutionContext context, Exception ex) public static void Error(this IExecutionContext context, Exception ex)
{ {
context.Error(ex.Message); context.Error(ex.Message);
@@ -889,6 +918,12 @@ namespace GitHub.Runner.Worker
context.AddIssue(new Issue() { Type = IssueType.Error, Message = message }); context.AddIssue(new Issue() { Type = IssueType.Error, Message = message });
} }
// Do not add a format string overload. See comment on ExecutionContext.Write().
public static void InfrastructureError(this IExecutionContext context, string message)
{
context.AddIssue(new Issue() { Type = IssueType.Error, Message = message, IsInfrastructureIssue = true});
}
// Do not add a format string overload. See comment on ExecutionContext.Write(). // Do not add a format string overload. See comment on ExecutionContext.Write().
public static void Warning(this IExecutionContext context, string message) public static void Warning(this IExecutionContext context, string message)
{ {
@@ -915,7 +950,7 @@ namespace GitHub.Runner.Worker
// Do not add a format string overload. See comment on ExecutionContext.Write(). // Do not add a format string overload. See comment on ExecutionContext.Write().
public static void Debug(this IExecutionContext context, string message) public static void Debug(this IExecutionContext context, string message)
{ {
if (context.WriteDebug) if (context.Global.WriteDebug)
{ {
var multilines = message?.Replace("\r\n", "\n")?.Split("\n"); var multilines = message?.Replace("\r\n", "\n")?.Split("\n");
if (multilines != null) if (multilines != null)
@@ -940,7 +975,7 @@ namespace GitHub.Runner.Worker
traceWriter = context.ToTemplateTraceWriter(); traceWriter = context.ToTemplateTraceWriter();
} }
var schema = PipelineTemplateSchemaFactory.GetSchema(); var schema = PipelineTemplateSchemaFactory.GetSchema();
return new PipelineTemplateEvaluator(traceWriter, schema, context.FileTable); return new PipelineTemplateEvaluator(traceWriter, schema, context.Global.FileTable);
} }
public static ObjectTemplating.ITraceWriter ToTemplateTraceWriter(this IExecutionContext context) public static ObjectTemplating.ITraceWriter ToTemplateTraceWriter(this IExecutionContext context)

View File

@@ -12,6 +12,8 @@ namespace GitHub.Runner.Worker.Expressions
{ {
public sealed class HashFilesFunction : Function public sealed class HashFilesFunction : Function
{ {
private const int _hashFileTimeoutSeconds = 120;
protected sealed override Object EvaluateCore( protected sealed override Object EvaluateCore(
EvaluationContext context, EvaluationContext context,
out ResultMemory resultMemory) out ResultMemory resultMemory)
@@ -89,19 +91,29 @@ namespace GitHub.Runner.Worker.Expressions
} }
env["patterns"] = string.Join(Environment.NewLine, patterns); env["patterns"] = string.Join(Environment.NewLine, patterns);
int exitCode = p.ExecuteAsync(workingDirectory: githubWorkspace, using (var tokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(_hashFileTimeoutSeconds)))
fileName: node,
arguments: $"\"{hashFilesScript.Replace("\"", "\\\"")}\"",
environment: env,
requireExitCodeZero: false,
cancellationToken: new CancellationTokenSource(TimeSpan.FromSeconds(120)).Token).GetAwaiter().GetResult();
if (exitCode != 0)
{ {
throw new InvalidOperationException($"hashFiles('{ExpressionUtility.StringEscape(string.Join(", ", patterns))}') failed. Fail to hash files under directory '{githubWorkspace}'"); try
} {
int exitCode = p.ExecuteAsync(workingDirectory: githubWorkspace,
fileName: node,
arguments: $"\"{hashFilesScript.Replace("\"", "\\\"")}\"",
environment: env,
requireExitCodeZero: false,
cancellationToken: tokenSource.Token).GetAwaiter().GetResult();
return hashResult; if (exitCode != 0)
{
throw new InvalidOperationException($"hashFiles('{ExpressionUtility.StringEscape(string.Join(", ", patterns))}') failed. Fail to hash files under directory '{githubWorkspace}'");
}
}
catch (OperationCanceledException) when (tokenSource.IsCancellationRequested)
{
throw new TimeoutException($"hashFiles('{ExpressionUtility.StringEscape(string.Join(", ", patterns))}') couldn't finish within {_hashFileTimeoutSeconds} seconds.");
}
return hashResult;
}
} }
private sealed class HashFilesTrace : ITraceWriter private sealed class HashFilesTrace : ITraceWriter

View File

@@ -0,0 +1,262 @@
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Worker.Container;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Text;
namespace GitHub.Runner.Worker
{
[ServiceLocator(Default = typeof(FileCommandManager))]
public interface IFileCommandManager : IRunnerService
{
void InitializeFiles(IExecutionContext context, ContainerInfo container);
void ProcessFiles(IExecutionContext context, ContainerInfo container);
}
public sealed class FileCommandManager : RunnerService, IFileCommandManager
{
private const string _folderName = "_runner_file_commands";
private List<IFileCommandExtension> _commandExtensions;
private string _fileSuffix = String.Empty;
private string _fileCommandDirectory;
private Tracing _trace;
public override void Initialize(IHostContext hostContext)
{
base.Initialize(hostContext);
_trace = HostContext.GetTrace(nameof(FileCommandManager));
_fileCommandDirectory = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Temp), _folderName);
if (!Directory.Exists(_fileCommandDirectory))
{
Directory.CreateDirectory(_fileCommandDirectory);
}
var extensionManager = hostContext.GetService<IExtensionManager>();
_commandExtensions = extensionManager.GetExtensions<IFileCommandExtension>() ?? new List<IFileCommandExtension>();
}
public void InitializeFiles(IExecutionContext context, ContainerInfo container)
{
var oldSuffix = _fileSuffix;
_fileSuffix = Guid.NewGuid().ToString();
foreach (var fileCommand in _commandExtensions)
{
var oldPath = Path.Combine(_fileCommandDirectory, fileCommand.FilePrefix + oldSuffix);
if (oldSuffix != String.Empty && File.Exists(oldPath))
{
TryDeleteFile(oldPath);
}
var newPath = Path.Combine(_fileCommandDirectory, fileCommand.FilePrefix + _fileSuffix);
TryDeleteFile(newPath);
File.Create(newPath).Dispose();
var pathToSet = container != null ? container.TranslateToContainerPath(newPath) : newPath;
context.SetGitHubContext(fileCommand.ContextName, pathToSet);
}
}
public void ProcessFiles(IExecutionContext context, ContainerInfo container)
{
foreach (var fileCommand in _commandExtensions)
{
try
{
fileCommand.ProcessCommand(context, Path.Combine(_fileCommandDirectory, fileCommand.FilePrefix + _fileSuffix),container);
}
catch (Exception ex)
{
context.Error($"Unable to process file command '{fileCommand.ContextName}' successfully.");
context.Error(ex);
context.CommandResult = TaskResult.Failed;
}
}
}
private bool TryDeleteFile(string path)
{
if (!File.Exists(path))
{
return true;
}
try
{
File.Delete(path);
}
catch (Exception e)
{
_trace.Warning($"Unable to delete file {path} for reason: {e.ToString()}");
return false;
}
return true;
}
}
public interface IFileCommandExtension : IExtension
{
string ContextName { get; }
string FilePrefix { get; }
void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container);
}
public sealed class AddPathFileCommand : RunnerService, IFileCommandExtension
{
public string ContextName => "path";
public string FilePrefix => "add_path_";
public Type ExtensionType => typeof(IFileCommandExtension);
public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container)
{
if (File.Exists(filePath))
{
var lines = File.ReadAllLines(filePath, Encoding.UTF8);
foreach(var line in lines)
{
if (line == string.Empty)
{
continue;
}
context.Global.PrependPath.RemoveAll(x => string.Equals(x, line, StringComparison.CurrentCulture));
context.Global.PrependPath.Add(line);
}
}
}
}
public sealed class SetEnvFileCommand : RunnerService, IFileCommandExtension
{
public string ContextName => "env";
public string FilePrefix => "set_env_";
public Type ExtensionType => typeof(IFileCommandExtension);
public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container)
{
try
{
var text = File.ReadAllText(filePath) ?? string.Empty;
var index = 0;
var line = ReadLine(text, ref index);
while (line != null)
{
if (!string.IsNullOrEmpty(line))
{
var equalsIndex = line.IndexOf("=", StringComparison.Ordinal);
var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal);
// Normal style NAME=VALUE
if (equalsIndex >= 0 && (heredocIndex < 0 || equalsIndex < heredocIndex))
{
var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None);
if (string.IsNullOrEmpty(line))
{
throw new Exception($"Invalid environment variable format '{line}'. Environment variable name must not be empty");
}
SetEnvironmentVariable(context, split[0], split[1]);
}
// Heredoc style NAME<<EOF
else if (heredocIndex >= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex))
{
var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None);
if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1]))
{
throw new Exception($"Invalid environment variable format '{line}'. Environment variable name must not be empty and delimiter must not be empty");
}
var name = split[0];
var delimiter = split[1];
var startIndex = index; // Start index of the value (inclusive)
var endIndex = index; // End index of the value (exclusive)
var tempLine = ReadLine(text, ref index, out var newline);
while (!string.Equals(tempLine, delimiter, StringComparison.Ordinal))
{
if (tempLine == null)
{
throw new Exception($"Invalid environment variable value. Matching delimiter not found '{delimiter}'");
}
endIndex = index - newline.Length;
tempLine = ReadLine(text, ref index, out newline);
}
var value = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty;
SetEnvironmentVariable(context, name, value);
}
else
{
throw new Exception($"Invalid environment variable format '{line}'");
}
}
line = ReadLine(text, ref index);
}
}
catch (DirectoryNotFoundException)
{
context.Debug($"Environment variables file does not exist '{filePath}'");
}
catch (FileNotFoundException)
{
context.Debug($"Environment variables file does not exist '{filePath}'");
}
}
private static void SetEnvironmentVariable(
IExecutionContext context,
string name,
string value)
{
context.Global.EnvironmentVariables[name] = value;
context.SetEnvContext(name, value);
context.Debug($"{name}='{value}'");
}
private static string ReadLine(
string text,
ref int index)
{
return ReadLine(text, ref index, out _);
}
private static string ReadLine(
string text,
ref int index,
out string newline)
{
if (index >= text.Length)
{
newline = null;
return null;
}
var originalIndex = index;
var lfIndex = text.IndexOf("\n", index, StringComparison.Ordinal);
if (lfIndex < 0)
{
index = text.Length;
newline = null;
return text.Substring(originalIndex);
}
#if OS_WINDOWS
var crLFIndex = text.IndexOf("\r\n", index, StringComparison.Ordinal);
if (crLFIndex >= 0 && crLFIndex < lfIndex)
{
index = crLFIndex + 2; // Skip over CRLF
newline = "\r\n";
return text.Substring(originalIndex, crLFIndex - originalIndex);
}
#endif
index = lfIndex + 1; // Skip over LF
newline = "\n";
return text.Substring(originalIndex, lfIndex - originalIndex);
}
}
}

View File

@@ -6,24 +6,30 @@ namespace GitHub.Runner.Worker
{ {
public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextData public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextData
{ {
private readonly HashSet<string> _contextEnvWhitelist = new HashSet<string>(StringComparer.OrdinalIgnoreCase) private readonly HashSet<string> _contextEnvAllowlist = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
{ {
"action", "action",
"action_path",
"action_ref",
"action_repository",
"actor", "actor",
"api_url", "api_url",
"base_ref", "base_ref",
"env",
"event_name", "event_name",
"event_path", "event_path",
"graphql_url", "graphql_url",
"head_ref", "head_ref",
"job", "job",
"path",
"ref", "ref",
"repository", "repository",
"repository_owner", "repository_owner",
"retention_days",
"run_id", "run_id",
"run_number", "run_number",
"server_url",
"sha", "sha",
"url",
"workflow", "workflow",
"workspace", "workspace",
}; };
@@ -32,11 +38,23 @@ namespace GitHub.Runner.Worker
{ {
foreach (var data in this) foreach (var data in this)
{ {
if (_contextEnvWhitelist.Contains(data.Key) && data.Value is StringContextData value) if (_contextEnvAllowlist.Contains(data.Key) && data.Value is StringContextData value)
{ {
yield return new KeyValuePair<string, string>($"GITHUB_{data.Key.ToUpperInvariant()}", value); yield return new KeyValuePair<string, string>($"GITHUB_{data.Key.ToUpperInvariant()}", value);
} }
} }
} }
public GitHubContext ShallowCopy()
{
var copy = new GitHubContext();
foreach (var pair in this)
{
copy[pair.Key] = pair.Value;
}
return copy;
}
} }
} }

View File

@@ -0,0 +1,24 @@
using System;
using System.Collections.Generic;
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common.Util;
using GitHub.Runner.Worker.Container;
namespace GitHub.Runner.Worker
{
public sealed class GlobalContext
{
public ContainerInfo Container { get; set; }
public List<ServiceEndpoint> Endpoints { get; set; }
public IDictionary<String, String> EnvironmentVariables { get; set; }
public PlanFeatures Features { get; set; }
public IList<String> FileTable { get; set; }
public IDictionary<String, IDictionary<String, String>> JobDefaults { get; set; }
public TaskOrchestrationPlanReference Plan { get; set; }
public List<string> PrependPath { get; set; }
public List<ContainerInfo> ServiceContainers { get; set; }
public StepsContext StepsContext { get; set; }
public Variables Variables { get; set; }
public bool WriteDebug { get; set; }
}
}

View File

@@ -0,0 +1,282 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Common;
using GitHub.Runner.Sdk;
using Pipelines = GitHub.DistributedTask.Pipelines;
namespace GitHub.Runner.Worker.Handlers
{
[ServiceLocator(Default = typeof(CompositeActionHandler))]
public interface ICompositeActionHandler : IHandler
{
CompositeActionExecutionData Data { get; set; }
}
public sealed class CompositeActionHandler : Handler, ICompositeActionHandler
{
public CompositeActionExecutionData Data { get; set; }
public async Task RunAsync(ActionRunStage stage)
{
// Validate args.
Trace.Entering();
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
ArgUtil.NotNull(Inputs, nameof(Inputs));
ArgUtil.NotNull(Data.Steps, nameof(Data.Steps));
// Resolve action steps
var actionSteps = Data.Steps;
// Create Context Data to reuse for each composite action step
var inputsData = new DictionaryContextData();
foreach (var i in Inputs)
{
inputsData[i.Key] = new StringContextData(i.Value);
}
// Initialize Composite Steps List of Steps
var compositeSteps = new List<IStep>();
// Temporary hack until after M271-ish. After M271-ish the server will never send an empty
// context name. Generated context names start with "__"
var childScopeName = ExecutionContext.GetFullyQualifiedContextName();
if (string.IsNullOrEmpty(childScopeName))
{
childScopeName = $"__{Guid.NewGuid()}";
}
foreach (Pipelines.ActionStep actionStep in actionSteps)
{
var actionRunner = HostContext.CreateService<IActionRunner>();
actionRunner.Action = actionStep;
actionRunner.Stage = stage;
actionRunner.Condition = actionStep.Condition;
var step = ExecutionContext.CreateCompositeStep(childScopeName, actionRunner, inputsData, Environment);
// Shallow copy github context
var gitHubContext = step.ExecutionContext.ExpressionValues["github"] as GitHubContext;
ArgUtil.NotNull(gitHubContext, nameof(gitHubContext));
gitHubContext = gitHubContext.ShallowCopy();
step.ExecutionContext.ExpressionValues["github"] = gitHubContext;
// Set GITHUB_ACTION_PATH
step.ExecutionContext.SetGitHubContext("action_path", ActionDirectory);
compositeSteps.Add(step);
}
try
{
// This is where we run each step.
await RunStepsAsync(compositeSteps);
// Get the pointer of the correct "steps" object and pass it to the ExecutionContext so that we can process the outputs correctly
ExecutionContext.ExpressionValues["inputs"] = inputsData;
ExecutionContext.ExpressionValues["steps"] = ExecutionContext.Global.StepsContext.GetScope(ExecutionContext.GetFullyQualifiedContextName());
ProcessCompositeActionOutputs();
ExecutionContext.Global.StepsContext.ClearScope(childScopeName);
}
catch (Exception ex)
{
// Composite StepRunner should never throw exception out.
Trace.Error($"Caught exception from composite steps {nameof(CompositeActionHandler)}: {ex}");
ExecutionContext.Error(ex);
ExecutionContext.Result = TaskResult.Failed;
}
}
private void ProcessCompositeActionOutputs()
{
ArgUtil.NotNull(ExecutionContext, nameof(ExecutionContext));
// Evaluate the mapped outputs value
if (Data.Outputs != null)
{
// Evaluate the outputs in the steps context to easily retrieve the values
var actionManifestManager = HostContext.GetService<IActionManifestManager>();
// Format ExpressionValues to Dictionary<string, PipelineContextData>
var evaluateContext = new Dictionary<string, PipelineContextData>(StringComparer.OrdinalIgnoreCase);
foreach (var pair in ExecutionContext.ExpressionValues)
{
evaluateContext[pair.Key] = pair.Value;
}
// Get the evluated composite outputs' values mapped to the outputs named
DictionaryContextData actionOutputs = actionManifestManager.EvaluateCompositeOutputs(ExecutionContext, Data.Outputs, evaluateContext);
// Set the outputs for the outputs object in the whole composite action
// Each pair is structured like this
// We ignore "description" for now
// {
// "the-output-name": {
// "description": "",
// "value": "the value"
// },
// ...
// }
foreach (var pair in actionOutputs)
{
var outputsName = pair.Key;
var outputsAttributes = pair.Value as DictionaryContextData;
outputsAttributes.TryGetValue("value", out var val);
if (val != null)
{
var outputsValue = val as StringContextData;
// Set output in the whole composite scope.
if (!String.IsNullOrEmpty(outputsValue))
{
ExecutionContext.SetOutput(outputsName, outputsValue, out _);
}
else
{
ExecutionContext.SetOutput(outputsName, "", out _);
}
}
}
}
}
private async Task RunStepsAsync(List<IStep> compositeSteps)
{
ArgUtil.NotNull(compositeSteps, nameof(compositeSteps));
// The parent StepsRunner of the whole Composite Action Step handles the cancellation stuff already.
foreach (IStep step in compositeSteps)
{
Trace.Info($"Processing composite step: DisplayName='{step.DisplayName}'");
step.ExecutionContext.ExpressionValues["steps"] = ExecutionContext.Global.StepsContext.GetScope(step.ExecutionContext.ScopeName);
// Populate env context for each step
Trace.Info("Initialize Env context for step");
#if OS_WINDOWS
var envContext = new DictionaryContextData();
#else
var envContext = new CaseSensitiveDictionaryContextData();
#endif
// Global env
foreach (var pair in ExecutionContext.Global.EnvironmentVariables)
{
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
// Stomps over with outside step env
if (step.ExecutionContext.ExpressionValues.TryGetValue("env", out var envContextData))
{
#if OS_WINDOWS
var dict = envContextData as DictionaryContextData;
#else
var dict = envContextData as CaseSensitiveDictionaryContextData;
#endif
foreach (var pair in dict)
{
envContext[pair.Key] = pair.Value;
}
}
step.ExecutionContext.ExpressionValues["env"] = envContext;
var actionStep = step as IActionRunner;
try
{
// Evaluate and merge action's env block to env context
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator();
var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, Common.Util.VarUtil.EnvironmentVariableKeyComparer);
foreach (var env in actionEnvironment)
{
envContext[env.Key] = new StringContextData(env.Value ?? string.Empty);
}
}
catch (Exception ex)
{
// fail the step since there is an evaluate error.
Trace.Info("Caught exception in Composite Steps Runner from expression for step.env");
// evaluateStepEnvFailed = true;
step.ExecutionContext.Error(ex);
step.ExecutionContext.Complete(TaskResult.Failed);
}
await RunStepAsync(step);
// Directly after the step, check if the step has failed or cancelled
// If so, return that to the output
if (step.ExecutionContext.Result == TaskResult.Failed || step.ExecutionContext.Result == TaskResult.Canceled)
{
ExecutionContext.Result = step.ExecutionContext.Result;
break;
}
// TODO: Add compat for other types of steps.
}
// Completion Status handled by StepsRunner for the whole Composite Action Step
}
private async Task RunStepAsync(IStep step)
{
// Start the step.
Trace.Info("Starting the step.");
step.ExecutionContext.Debug($"Starting: {step.DisplayName}");
// TODO: Fix for Step Level Timeout Attributes for an individual Composite Run Step
// For now, we are not going to support this for an individual composite run step
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator();
await Common.Util.EncodingUtil.SetEncoding(HostContext, Trace, step.ExecutionContext.CancellationToken);
try
{
await step.RunAsync();
}
catch (OperationCanceledException ex)
{
if (step.ExecutionContext.CancellationToken.IsCancellationRequested &&
!ExecutionContext.Root.CancellationToken.IsCancellationRequested)
{
Trace.Error($"Caught timeout exception from step: {ex.Message}");
step.ExecutionContext.Error("The action has timed out.");
step.ExecutionContext.Result = TaskResult.Failed;
}
else
{
Trace.Error($"Caught cancellation exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Canceled;
}
}
catch (Exception ex)
{
// Log the error and fail the step.
Trace.Error($"Caught exception from step: {ex}");
step.ExecutionContext.Error(ex);
step.ExecutionContext.Result = TaskResult.Failed;
}
// Merge execution context result with command result
if (step.ExecutionContext.CommandResult != null)
{
step.ExecutionContext.Result = Common.Util.TaskResultUtil.MergeTaskResults(step.ExecutionContext.Result, step.ExecutionContext.CommandResult.Value);
}
Trace.Info($"Step result: {step.ExecutionContext.Result}");
// Complete the step context.
step.ExecutionContext.Debug($"Finishing: {step.DisplayName}");
}
}
}

View File

@@ -49,8 +49,9 @@ namespace GitHub.Runner.Worker.Handlers
// ensure docker file exist // ensure docker file exist
var dockerFile = Path.Combine(ActionDirectory, Data.Image); var dockerFile = Path.Combine(ActionDirectory, Data.Image);
ArgUtil.File(dockerFile, nameof(Data.Image)); ArgUtil.File(dockerFile, nameof(Data.Image));
ExecutionContext.Output($"Dockerfile for action: '{dockerFile}'.");
ExecutionContext.Output($"##[group]Building docker image");
ExecutionContext.Output($"Dockerfile for action: '{dockerFile}'.");
var imageName = $"{dockerManger.DockerInstanceLabel}:{ExecutionContext.Id.ToString("N")}"; var imageName = $"{dockerManger.DockerInstanceLabel}:{ExecutionContext.Id.ToString("N")}";
var buildExitCode = await dockerManger.DockerBuild( var buildExitCode = await dockerManger.DockerBuild(
ExecutionContext, ExecutionContext,
@@ -58,6 +59,8 @@ namespace GitHub.Runner.Worker.Handlers
dockerFile, dockerFile,
Directory.GetParent(dockerFile).FullName, Directory.GetParent(dockerFile).FullName,
imageName); imageName);
ExecutionContext.Output("##[endgroup]");
if (buildExitCode != 0) if (buildExitCode != 0)
{ {
throw new InvalidOperationException($"Docker build failed with exit code {buildExitCode}"); throw new InvalidOperationException($"Docker build failed with exit code {buildExitCode}");
@@ -67,7 +70,7 @@ namespace GitHub.Runner.Worker.Handlers
} }
// run container // run container
var container = new ContainerInfo() var container = new ContainerInfo(HostContext)
{ {
ContainerImage = Data.Image, ContainerImage = Data.Image,
ContainerName = ExecutionContext.Id.ToString("N"), ContainerName = ExecutionContext.Id.ToString("N"),
@@ -158,16 +161,21 @@ namespace GitHub.Runner.Worker.Handlers
Directory.CreateDirectory(tempHomeDirectory); Directory.CreateDirectory(tempHomeDirectory);
this.Environment["HOME"] = tempHomeDirectory; this.Environment["HOME"] = tempHomeDirectory;
var tempFileCommandDirectory = Path.Combine(tempDirectory, "_runner_file_commands");
ArgUtil.Directory(tempFileCommandDirectory, nameof(tempFileCommandDirectory));
var tempWorkflowDirectory = Path.Combine(tempDirectory, "_github_workflow"); var tempWorkflowDirectory = Path.Combine(tempDirectory, "_github_workflow");
ArgUtil.Directory(tempWorkflowDirectory, nameof(tempWorkflowDirectory)); ArgUtil.Directory(tempWorkflowDirectory, nameof(tempWorkflowDirectory));
container.MountVolumes.Add(new MountVolume("/var/run/docker.sock", "/var/run/docker.sock")); container.MountVolumes.Add(new MountVolume("/var/run/docker.sock", "/var/run/docker.sock"));
container.MountVolumes.Add(new MountVolume(tempHomeDirectory, "/github/home")); container.MountVolumes.Add(new MountVolume(tempHomeDirectory, "/github/home"));
container.MountVolumes.Add(new MountVolume(tempWorkflowDirectory, "/github/workflow")); container.MountVolumes.Add(new MountVolume(tempWorkflowDirectory, "/github/workflow"));
container.MountVolumes.Add(new MountVolume(tempFileCommandDirectory, "/github/file_commands"));
container.MountVolumes.Add(new MountVolume(defaultWorkingDirectory, "/github/workspace")); container.MountVolumes.Add(new MountVolume(defaultWorkingDirectory, "/github/workspace"));
container.AddPathTranslateMapping(tempHomeDirectory, "/github/home"); container.AddPathTranslateMapping(tempHomeDirectory, "/github/home");
container.AddPathTranslateMapping(tempWorkflowDirectory, "/github/workflow"); container.AddPathTranslateMapping(tempWorkflowDirectory, "/github/workflow");
container.AddPathTranslateMapping(tempFileCommandDirectory, "/github/file_commands");
container.AddPathTranslateMapping(defaultWorkingDirectory, "/github/workspace"); container.AddPathTranslateMapping(defaultWorkingDirectory, "/github/workspace");
container.ContainerWorkDirectory = "/github/workspace"; container.ContainerWorkDirectory = "/github/workspace";
@@ -185,7 +193,7 @@ namespace GitHub.Runner.Worker.Handlers
} }
// Add Actions Runtime server info // Add Actions Runtime server info
var systemConnection = ExecutionContext.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); var systemConnection = ExecutionContext.Global.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
Environment["ACTIONS_RUNTIME_URL"] = systemConnection.Url.AbsoluteUri; Environment["ACTIONS_RUNTIME_URL"] = systemConnection.Url.AbsoluteUri;
Environment["ACTIONS_RUNTIME_TOKEN"] = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken]; Environment["ACTIONS_RUNTIME_TOKEN"] = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken];
if (systemConnection.Data.TryGetValue("CacheServerUrl", out var cacheUrl) && !string.IsNullOrEmpty(cacheUrl)) if (systemConnection.Data.TryGetValue("CacheServerUrl", out var cacheUrl) && !string.IsNullOrEmpty(cacheUrl))

View File

@@ -148,14 +148,14 @@ namespace GitHub.Runner.Worker.Handlers
{ {
// Validate args. // Validate args.
Trace.Entering(); Trace.Entering();
ArgUtil.NotNull(ExecutionContext.PrependPath, nameof(ExecutionContext.PrependPath)); ArgUtil.NotNull(ExecutionContext.Global.PrependPath, nameof(ExecutionContext.Global.PrependPath));
if (ExecutionContext.PrependPath.Count == 0) if (ExecutionContext.Global.PrependPath.Count == 0)
{ {
return; return;
} }
// Prepend path. // Prepend path.
string prepend = string.Join(Path.PathSeparator.ToString(), ExecutionContext.PrependPath.Reverse<string>()); string prepend = string.Join(Path.PathSeparator.ToString(), ExecutionContext.Global.PrependPath.Reverse<string>());
var containerStepHost = StepHost as ContainerStepHost; var containerStepHost = StepHost as ContainerStepHost;
if (containerStepHost != null) if (containerStepHost != null)
{ {

View File

@@ -66,6 +66,11 @@ namespace GitHub.Runner.Worker.Handlers
handler = HostContext.CreateService<IRunnerPluginHandler>(); handler = HostContext.CreateService<IRunnerPluginHandler>();
(handler as IRunnerPluginHandler).Data = data as PluginActionExecutionData; (handler as IRunnerPluginHandler).Data = data as PluginActionExecutionData;
} }
else if (data.ExecutionType == ActionExecutionType.Composite)
{
handler = HostContext.CreateService<ICompositeActionHandler>();
(handler as ICompositeActionHandler).Data = data as CompositeActionExecutionData;
}
else else
{ {
// This should never happen. // This should never happen.

View File

@@ -46,7 +46,7 @@ namespace GitHub.Runner.Worker.Handlers
} }
// Add Actions Runtime server info // Add Actions Runtime server info
var systemConnection = ExecutionContext.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); var systemConnection = ExecutionContext.Global.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase));
Environment["ACTIONS_RUNTIME_URL"] = systemConnection.Url.AbsoluteUri; Environment["ACTIONS_RUNTIME_URL"] = systemConnection.Url.AbsoluteUri;
Environment["ACTIONS_RUNTIME_TOKEN"] = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken]; Environment["ACTIONS_RUNTIME_TOKEN"] = systemConnection.Authorization.Parameters[EndpointAuthorizationParameters.AccessToken];
if (systemConnection.Data.TryGetValue("CacheServerUrl", out var cacheUrl) && !string.IsNullOrEmpty(cacheUrl)) if (systemConnection.Data.TryGetValue("CacheServerUrl", out var cacheUrl) && !string.IsNullOrEmpty(cacheUrl))
@@ -113,7 +113,7 @@ namespace GitHub.Runner.Worker.Handlers
requireExitCodeZero: false, requireExitCodeZero: false,
outputEncoding: outputEncoding, outputEncoding: outputEncoding,
killProcessOnCancel: false, killProcessOnCancel: false,
inheritConsoleHandler: !ExecutionContext.Variables.Retain_Default_Encoding, inheritConsoleHandler: !ExecutionContext.Global.Variables.Retain_Default_Encoding,
cancellationToken: ExecutionContext.CancellationToken); cancellationToken: ExecutionContext.CancellationToken);
// Wait for either the node exit or force finish through ##vso command // Wait for either the node exit or force finish through ##vso command

View File

@@ -31,7 +31,7 @@ namespace GitHub.Runner.Worker.Handlers
{ {
_executionContext = executionContext; _executionContext = executionContext;
_commandManager = commandManager; _commandManager = commandManager;
_container = container ?? executionContext.Container; _container = container ?? executionContext.Global.Container;
// Recursion failsafe (test override) // Recursion failsafe (test override)
var failsafeString = Environment.GetEnvironmentVariable("RUNNER_TEST_GET_REPOSITORY_PATH_FAILSAFE"); var failsafeString = Environment.GetEnvironmentVariable("RUNNER_TEST_GET_REPOSITORY_PATH_FAILSAFE");
@@ -41,7 +41,7 @@ namespace GitHub.Runner.Worker.Handlers
} }
// Determine the timeout // Determine the timeout
var timeoutStr = _executionContext.Variables.Get(_timeoutKey); var timeoutStr = _executionContext.Global.Variables.Get(_timeoutKey);
if (string.IsNullOrEmpty(timeoutStr) || if (string.IsNullOrEmpty(timeoutStr) ||
!TimeSpan.TryParse(timeoutStr, CultureInfo.InvariantCulture, out _timeout) || !TimeSpan.TryParse(timeoutStr, CultureInfo.InvariantCulture, out _timeout) ||
_timeout <= TimeSpan.Zero) _timeout <= TimeSpan.Zero)
@@ -352,15 +352,24 @@ namespace GitHub.Runner.Worker.Handlers
if (File.Exists(gitConfigPath)) if (File.Exists(gitConfigPath))
{ {
// Check if the config contains the workflow repository url // Check if the config contains the workflow repository url
var qualifiedRepository = _executionContext.GetGitHubContext("repository"); var serverUrl = _executionContext.GetGitHubContext("server_url");
var configMatch = $"url = https://github.com/{qualifiedRepository}"; serverUrl = !string.IsNullOrEmpty(serverUrl) ? serverUrl : "https://github.com";
var host = new Uri(serverUrl, UriKind.Absolute).Host;
var nameWithOwner = _executionContext.GetGitHubContext("repository");
var patterns = new[] {
$"url = {serverUrl}/{nameWithOwner}",
$"url = git@{host}:{nameWithOwner}.git",
};
var content = File.ReadAllText(gitConfigPath); var content = File.ReadAllText(gitConfigPath);
foreach (var line in content.Split("\n").Select(x => x.Trim())) foreach (var line in content.Split("\n").Select(x => x.Trim()))
{ {
if (String.Equals(line, configMatch, StringComparison.OrdinalIgnoreCase)) foreach (var pattern in patterns)
{ {
repositoryPath = directoryPath; if (String.Equals(line, pattern, StringComparison.OrdinalIgnoreCase))
break; {
repositoryPath = directoryPath;
break;
}
} }
} }
} }

View File

@@ -23,6 +23,19 @@ namespace GitHub.Runner.Worker.Handlers
public override void PrintActionDetails(ActionRunStage stage) public override void PrintActionDetails(ActionRunStage stage)
{ {
// We don't want to display the internal workings if composite (similar/equivalent information can be found in debug)
void writeDetails(string message)
{
if (ExecutionContext.InsideComposite)
{
ExecutionContext.Debug(message);
}
else
{
ExecutionContext.Output(message);
}
}
if (stage == ActionRunStage.Post) if (stage == ActionRunStage.Post)
{ {
throw new NotSupportedException("Script action should not have 'Post' job action."); throw new NotSupportedException("Script action should not have 'Post' job action.");
@@ -39,7 +52,7 @@ namespace GitHub.Runner.Worker.Handlers
firstLine = firstLine.Substring(0, firstNewLine); firstLine = firstLine.Substring(0, firstNewLine);
} }
ExecutionContext.Output($"##[group]Run {firstLine}"); writeDetails(ExecutionContext.InsideComposite ? $"Run {firstLine}" : $"##[group]Run {firstLine}");
} }
else else
{ {
@@ -50,20 +63,20 @@ namespace GitHub.Runner.Worker.Handlers
foreach (var line in multiLines) foreach (var line in multiLines)
{ {
// Bright Cyan color // Bright Cyan color
ExecutionContext.Output($"\x1b[36;1m{line}\x1b[0m"); writeDetails($"\x1b[36;1m{line}\x1b[0m");
} }
string argFormat; string argFormat;
string shellCommand; string shellCommand;
string shellCommandPath = null; string shellCommandPath = null;
bool validateShellOnHost = !(StepHost is ContainerStepHost); bool validateShellOnHost = !(StepHost is ContainerStepHost);
string prependPath = string.Join(Path.PathSeparator.ToString(), ExecutionContext.PrependPath.Reverse<string>()); string prependPath = string.Join(Path.PathSeparator.ToString(), ExecutionContext.Global.PrependPath.Reverse<string>());
string shell = null; string shell = null;
if (!Inputs.TryGetValue("shell", out shell) || string.IsNullOrEmpty(shell)) if (!Inputs.TryGetValue("shell", out shell) || string.IsNullOrEmpty(shell))
{ {
// TODO: figure out how defaults interact with template later // TODO: figure out how defaults interact with template later
// for now, we won't check job.defaults if we are inside a template. // for now, we won't check job.defaults if we are inside a template.
if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.JobDefaults.TryGetValue("run", out var runDefaults)) if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults))
{ {
runDefaults.TryGetValue("shell", out shell); runDefaults.TryGetValue("shell", out shell);
} }
@@ -109,23 +122,23 @@ namespace GitHub.Runner.Worker.Handlers
if (!string.IsNullOrEmpty(shellCommandPath)) if (!string.IsNullOrEmpty(shellCommandPath))
{ {
ExecutionContext.Output($"shell: {shellCommandPath} {argFormat}"); writeDetails($"shell: {shellCommandPath} {argFormat}");
} }
else else
{ {
ExecutionContext.Output($"shell: {shellCommand} {argFormat}"); writeDetails($"shell: {shellCommand} {argFormat}");
} }
if (this.Environment?.Count > 0) if (this.Environment?.Count > 0)
{ {
ExecutionContext.Output("env:"); writeDetails("env:");
foreach (var env in this.Environment) foreach (var env in this.Environment)
{ {
ExecutionContext.Output($" {env.Key}: {env.Value}"); writeDetails($" {env.Key}: {env.Value}");
} }
} }
ExecutionContext.Output("##[endgroup]"); writeDetails(ExecutionContext.InsideComposite ? "" : "##[endgroup]");
} }
public async Task RunAsync(ActionRunStage stage) public async Task RunAsync(ActionRunStage stage)
@@ -151,9 +164,7 @@ namespace GitHub.Runner.Worker.Handlers
string workingDirectory = null; string workingDirectory = null;
if (!Inputs.TryGetValue("workingDirectory", out workingDirectory)) if (!Inputs.TryGetValue("workingDirectory", out workingDirectory))
{ {
// TODO: figure out how defaults interact with template later if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults))
// for now, we won't check job.defaults if we are inside a template.
if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.JobDefaults.TryGetValue("run", out var runDefaults))
{ {
if (runDefaults.TryGetValue("working-directory", out workingDirectory)) if (runDefaults.TryGetValue("working-directory", out workingDirectory))
{ {
@@ -167,9 +178,7 @@ namespace GitHub.Runner.Worker.Handlers
string shell = null; string shell = null;
if (!Inputs.TryGetValue("shell", out shell) || string.IsNullOrEmpty(shell)) if (!Inputs.TryGetValue("shell", out shell) || string.IsNullOrEmpty(shell))
{ {
// TODO: figure out how defaults interact with template later if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults))
// for now, we won't check job.defaults if we are inside a template.
if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.JobDefaults.TryGetValue("run", out var runDefaults))
{ {
if (runDefaults.TryGetValue("shell", out shell)) if (runDefaults.TryGetValue("shell", out shell))
{ {
@@ -180,7 +189,7 @@ namespace GitHub.Runner.Worker.Handlers
var isContainerStepHost = StepHost is ContainerStepHost; var isContainerStepHost = StepHost is ContainerStepHost;
string prependPath = string.Join(Path.PathSeparator.ToString(), ExecutionContext.PrependPath.Reverse<string>()); string prependPath = string.Join(Path.PathSeparator.ToString(), ExecutionContext.Global.PrependPath.Reverse<string>());
string commandPath, argFormat, shellCommand; string commandPath, argFormat, shellCommand;
// Set up default command and arguments // Set up default command and arguments
if (string.IsNullOrEmpty(shell)) if (string.IsNullOrEmpty(shell))
@@ -232,7 +241,7 @@ namespace GitHub.Runner.Worker.Handlers
#if OS_WINDOWS #if OS_WINDOWS
// Normalize Windows line endings // Normalize Windows line endings
contents = contents.Replace("\r\n", "\n").Replace("\n", "\r\n"); contents = contents.Replace("\r\n", "\n").Replace("\n", "\r\n");
var encoding = ExecutionContext.Variables.Retain_Default_Encoding && Console.InputEncoding.CodePage != 65001 var encoding = ExecutionContext.Global.Variables.Retain_Default_Encoding && Console.InputEncoding.CodePage != 65001
? Console.InputEncoding ? Console.InputEncoding
: new UTF8Encoding(false); : new UTF8Encoding(false);
#else #else
@@ -285,7 +294,7 @@ namespace GitHub.Runner.Worker.Handlers
requireExitCodeZero: false, requireExitCodeZero: false,
outputEncoding: null, outputEncoding: null,
killProcessOnCancel: false, killProcessOnCancel: false,
inheritConsoleHandler: !ExecutionContext.Variables.Retain_Default_Encoding, inheritConsoleHandler: !ExecutionContext.Global.Variables.Retain_Default_Encoding,
cancellationToken: ExecutionContext.CancellationToken); cancellationToken: ExecutionContext.CancellationToken);
// Error // Error

View File

@@ -5,6 +5,7 @@ using System.Globalization;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Runtime.Serialization; using System.Runtime.Serialization;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.Expressions2;
using GitHub.DistributedTask.ObjectTemplating.Tokens; using GitHub.DistributedTask.ObjectTemplating.Tokens;
@@ -41,6 +42,8 @@ namespace GitHub.Runner.Worker
private readonly HashSet<string> _existingProcesses = new HashSet<string>(StringComparer.OrdinalIgnoreCase); private readonly HashSet<string> _existingProcesses = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
private bool _processCleanup; private bool _processCleanup;
private string _processLookupId = $"github_{Guid.NewGuid()}"; private string _processLookupId = $"github_{Guid.NewGuid()}";
private CancellationTokenSource _diskSpaceCheckToken = new CancellationTokenSource();
private Task _diskSpaceCheckTask = null;
// Download all required actions. // Download all required actions.
// Make sure all condition inputs are valid. // Make sure all condition inputs are valid.
@@ -64,6 +67,24 @@ namespace GitHub.Runner.Worker
context.Debug($"Starting: Set up job"); context.Debug($"Starting: Set up job");
context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'"); context.Output($"Current runner version: '{BuildConstants.RunnerPackage.Version}'");
var setting = HostContext.GetService<IConfigurationStore>().GetSettings();
var credFile = HostContext.GetConfigFile(WellKnownConfigFile.Credentials);
if (File.Exists(credFile))
{
var credData = IOUtil.LoadObject<CredentialData>(credFile);
if (credData != null &&
credData.Data.TryGetValue("clientId", out var clientId))
{
// print out HostName for self-hosted runner
context.Output($"Runner name: '{setting.AgentName}'");
if (message.Variables.TryGetValue("system.runnerGroupName", out VariableValue runnerGroupName))
{
context.Output($"Runner group name: '{runnerGroupName.Value}'");
}
context.Output($"Machine name: '{Environment.MachineName}'");
}
}
var setupInfoFile = HostContext.GetConfigFile(WellKnownConfigFile.SetupInfo); var setupInfoFile = HostContext.GetConfigFile(WellKnownConfigFile.SetupInfo);
if (File.Exists(setupInfoFile)) if (File.Exists(setupInfoFile))
{ {
@@ -131,12 +152,13 @@ namespace GitHub.Runner.Worker
// Temporary hack for GHES alpha // Temporary hack for GHES alpha
var configurationStore = HostContext.GetService<IConfigurationStore>(); var configurationStore = HostContext.GetService<IConfigurationStore>();
var runnerSettings = configurationStore.GetSettings(); var runnerSettings = configurationStore.GetSettings();
if (string.IsNullOrEmpty(context.GetGitHubContext("url")) && !runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl)) if (string.IsNullOrEmpty(context.GetGitHubContext("server_url")) && !runnerSettings.IsHostedServer && !string.IsNullOrEmpty(runnerSettings.GitHubUrl))
{ {
var url = new Uri(runnerSettings.GitHubUrl); var url = new Uri(runnerSettings.GitHubUrl);
var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}"; var portInfo = url.IsDefaultPort ? string.Empty : $":{url.Port.ToString(CultureInfo.InvariantCulture)}";
context.SetGitHubContext("url", $"{url.Scheme}://{url.Host}{portInfo}"); context.SetGitHubContext("server_url", $"{url.Scheme}://{url.Host}{portInfo}");
context.SetGitHubContext("api_url", $"{url.Scheme}://{url.Host}{portInfo}/api/v3"); context.SetGitHubContext("api_url", $"{url.Scheme}://{url.Host}{portInfo}/api/v3");
context.SetGitHubContext("graphql_url", $"{url.Scheme}://{url.Host}{portInfo}/api/graphql");
} }
// Evaluate the job-level environment variables // Evaluate the job-level environment variables
@@ -147,7 +169,7 @@ namespace GitHub.Runner.Worker
var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, jobContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer); var environmentVariables = templateEvaluator.EvaluateStepEnvironment(token, jobContext.ExpressionValues, jobContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer);
foreach (var pair in environmentVariables) foreach (var pair in environmentVariables)
{ {
context.EnvironmentVariables[pair.Key] = pair.Value ?? string.Empty; context.Global.EnvironmentVariables[pair.Key] = pair.Value ?? string.Empty;
context.SetEnvContext(pair.Key, pair.Value ?? string.Empty); context.SetEnvContext(pair.Key, pair.Value ?? string.Empty);
} }
} }
@@ -157,7 +179,7 @@ namespace GitHub.Runner.Worker
var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues, jobContext.ExpressionFunctions); var container = templateEvaluator.EvaluateJobContainer(message.JobContainer, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
if (container != null) if (container != null)
{ {
jobContext.Container = new Container.ContainerInfo(HostContext, container); jobContext.Global.Container = new Container.ContainerInfo(HostContext, container);
} }
// Evaluate the job service containers // Evaluate the job service containers
@@ -169,7 +191,7 @@ namespace GitHub.Runner.Worker
{ {
var networkAlias = pair.Key; var networkAlias = pair.Key;
var serviceContainer = pair.Value; var serviceContainer = pair.Value;
jobContext.ServiceContainers.Add(new Container.ContainerInfo(HostContext, serviceContainer, false, networkAlias)); jobContext.Global.ServiceContainers.Add(new Container.ContainerInfo(HostContext, serviceContainer, false, networkAlias));
} }
} }
@@ -180,14 +202,14 @@ namespace GitHub.Runner.Worker
var defaults = token.AssertMapping("defaults"); var defaults = token.AssertMapping("defaults");
if (defaults.Any(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase))) if (defaults.Any(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase)))
{ {
context.JobDefaults["run"] = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); context.Global.JobDefaults["run"] = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
var defaultsRun = defaults.First(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase)); var defaultsRun = defaults.First(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase));
var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsRun.Value, jobContext.ExpressionValues, jobContext.ExpressionFunctions); var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsRun.Value, jobContext.ExpressionValues, jobContext.ExpressionFunctions);
foreach (var pair in jobDefaults) foreach (var pair in jobDefaults)
{ {
if (!string.IsNullOrEmpty(pair.Value)) if (!string.IsNullOrEmpty(pair.Value))
{ {
context.JobDefaults["run"][pair.Key] = pair.Value; context.Global.JobDefaults["run"][pair.Key] = pair.Value;
} }
} }
} }
@@ -201,15 +223,15 @@ namespace GitHub.Runner.Worker
preJobSteps.AddRange(prepareResult.ContainerSetupSteps); preJobSteps.AddRange(prepareResult.ContainerSetupSteps);
// Add start-container steps, record and stop-container steps // Add start-container steps, record and stop-container steps
if (jobContext.Container != null || jobContext.ServiceContainers.Count > 0) if (jobContext.Global.Container != null || jobContext.Global.ServiceContainers.Count > 0)
{ {
var containerProvider = HostContext.GetService<IContainerOperationProvider>(); var containerProvider = HostContext.GetService<IContainerOperationProvider>();
var containers = new List<Container.ContainerInfo>(); var containers = new List<Container.ContainerInfo>();
if (jobContext.Container != null) if (jobContext.Global.Container != null)
{ {
containers.Add(jobContext.Container); containers.Add(jobContext.Global.Container);
} }
containers.AddRange(jobContext.ServiceContainers); containers.AddRange(jobContext.Global.ServiceContainers);
preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync, preJobSteps.Add(new JobExtensionRunner(runAsync: containerProvider.StartContainersAsync,
condition: $"{PipelineTemplateConstants.Success}()", condition: $"{PipelineTemplateConstants.Success}()",
@@ -281,7 +303,7 @@ namespace GitHub.Runner.Worker
{ {
ArgUtil.NotNull(actionStep, step.DisplayName); ArgUtil.NotNull(actionStep, step.DisplayName);
intraActionStates.TryGetValue(actionStep.Action.Id, out var intraActionState); intraActionStates.TryGetValue(actionStep.Action.Id, out var intraActionState);
actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, actionStep.Action.ScopeName, actionStep.Action.ContextName, intraActionState); actionStep.ExecutionContext = jobContext.CreateChild(actionStep.Action.Id, actionStep.DisplayName, actionStep.Action.Name, null, actionStep.Action.ContextName, intraActionState);
} }
} }
@@ -290,7 +312,7 @@ namespace GitHub.Runner.Worker
steps.AddRange(jobSteps); steps.AddRange(jobSteps);
// Prepare for orphan process cleanup // Prepare for orphan process cleanup
_processCleanup = jobContext.Variables.GetBoolean("process.clean") ?? true; _processCleanup = jobContext.Global.Variables.GetBoolean("process.clean") ?? true;
if (_processCleanup) if (_processCleanup)
{ {
// Set the RUNNER_TRACKING_ID env variable. // Set the RUNNER_TRACKING_ID env variable.
@@ -306,6 +328,12 @@ namespace GitHub.Runner.Worker
} }
} }
jobContext.Global.EnvironmentVariables.TryGetValue(Constants.Runner.Features.DiskSpaceWarning, out var enableWarning);
if (StringUtil.ConvertToBoolean(enableWarning, defaultValue: true))
{
_diskSpaceCheckTask = CheckDiskSpaceAsync(context, _diskSpaceCheckToken.Token);
}
return steps; return steps;
} }
catch (OperationCanceledException ex) when (jobContext.CancellationToken.IsCancellationRequested) catch (OperationCanceledException ex) when (jobContext.CancellationToken.IsCancellationRequested)
@@ -316,6 +344,14 @@ namespace GitHub.Runner.Worker
context.Result = TaskResult.Canceled; context.Result = TaskResult.Canceled;
throw; throw;
} }
catch (FailedToResolveActionDownloadInfoException ex)
{
// Log the error and fail the JobExtension Initialization.
Trace.Error($"Caught exception from JobExtenion Initialization: {ex}");
context.InfrastructureError(ex.Message);
context.Result = TaskResult.Failed;
throw;
}
catch (Exception ex) catch (Exception ex)
{ {
// Log the error and fail the JobExtension Initialization. // Log the error and fail the JobExtension Initialization.
@@ -346,6 +382,24 @@ namespace GitHub.Runner.Worker
context.Start(); context.Start();
context.Debug("Starting: Complete job"); context.Debug("Starting: Complete job");
Trace.Info("Initialize Env context");
#if OS_WINDOWS
var envContext = new DictionaryContextData();
#else
var envContext = new CaseSensitiveDictionaryContextData();
#endif
context.ExpressionValues["env"] = envContext;
foreach (var pair in context.Global.EnvironmentVariables)
{
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
// Populate env context for each step
Trace.Info("Initialize steps context");
context.ExpressionValues["steps"] = context.Global.StepsContext.GetScope(context.ScopeName);
var templateEvaluator = context.ToPipelineTemplateEvaluator();
// Evaluate job outputs // Evaluate job outputs
if (message.JobOutputs != null && message.JobOutputs.Type != TokenType.Null) if (message.JobOutputs != null && message.JobOutputs.Type != TokenType.Null)
{ {
@@ -355,21 +409,7 @@ namespace GitHub.Runner.Worker
// Populate env context for each step // Populate env context for each step
Trace.Info("Initialize Env context for evaluating job outputs"); Trace.Info("Initialize Env context for evaluating job outputs");
#if OS_WINDOWS
var envContext = new DictionaryContextData();
#else
var envContext = new CaseSensitiveDictionaryContextData();
#endif
context.ExpressionValues["env"] = envContext;
foreach (var pair in context.EnvironmentVariables)
{
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
Trace.Info("Initialize steps context for evaluating job outputs");
context.ExpressionValues["steps"] = context.StepsContext.GetScope(context.ScopeName);
var templateEvaluator = context.ToPipelineTemplateEvaluator();
var outputs = templateEvaluator.EvaluateJobOutput(message.JobOutputs, context.ExpressionValues, context.ExpressionFunctions); var outputs = templateEvaluator.EvaluateJobOutput(message.JobOutputs, context.ExpressionValues, context.ExpressionFunctions);
foreach (var output in outputs) foreach (var output in outputs)
{ {
@@ -398,7 +438,35 @@ namespace GitHub.Runner.Worker
} }
} }
if (context.Variables.GetBoolean(Constants.Variables.Actions.RunnerDebug) ?? false) // Evaluate environment data
if (jobContext.ActionsEnvironment?.Url != null && jobContext.ActionsEnvironment?.Url.Type != TokenType.Null)
{
try
{
context.Output($"Evaluate and set environment url");
var environmentUrlToken = templateEvaluator.EvaluateEnvironmentUrl(jobContext.ActionsEnvironment.Url, context.ExpressionValues, context.ExpressionFunctions);
var environmentUrl = environmentUrlToken.AssertString("environment.url");
if (!string.Equals(environmentUrl.Value, HostContext.SecretMasker.MaskSecrets(environmentUrl.Value)))
{
context.Warning($"Skip setting environment url as environment '{jobContext.ActionsEnvironment.Name}' may contain secret.");
}
else
{
context.Output($"Evaluated environment url: {environmentUrl}");
jobContext.ActionsEnvironment.Url = environmentUrlToken;
}
}
catch (Exception ex)
{
context.Result = TaskResult.Failed;
context.Error($"Failed to evaluate environment url");
context.Error(ex);
jobContext.Result = TaskResultUtil.MergeTaskResults(jobContext.Result, TaskResult.Failed);
}
}
if (context.Global.Variables.GetBoolean(Constants.Variables.Actions.RunnerDebug) ?? false)
{ {
Trace.Info("Support log upload starting."); Trace.Info("Support log upload starting.");
context.Output("Uploading runner diagnostic logs"); context.Output("Uploading runner diagnostic logs");
@@ -470,6 +538,11 @@ namespace GitHub.Runner.Worker
} }
} }
} }
if (_diskSpaceCheckTask != null)
{
_diskSpaceCheckToken.Cancel();
}
} }
catch (Exception ex) catch (Exception ex)
{ {
@@ -485,6 +558,39 @@ namespace GitHub.Runner.Worker
} }
} }
private async Task CheckDiskSpaceAsync(IExecutionContext context, CancellationToken token)
{
while (!token.IsCancellationRequested)
{
// Add warning when disk is lower than system.runner.lowdiskspacethreshold from service (default to 100 MB on service side)
var lowDiskSpaceThreshold = context.Global.Variables.GetInt(WellKnownDistributedTaskVariables.RunnerLowDiskspaceThreshold);
if (lowDiskSpaceThreshold == null)
{
Trace.Info($"Low diskspace warning is not enabled.");
return;
}
var workDirRoot = Directory.GetDirectoryRoot(HostContext.GetDirectory(WellKnownDirectory.Work));
var driveInfo = new DriveInfo(workDirRoot);
var freeSpaceInMB = driveInfo.AvailableFreeSpace / 1024 / 1024;
if (freeSpaceInMB < lowDiskSpaceThreshold)
{
var issue = new Issue() { Type = IssueType.Warning, Message = $"You are running out of disk space. The runner will stop working when the machine runs out of disk space. Free space left: {freeSpaceInMB} MB" };
issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.LowDiskSpace;
context.AddIssue(issue);
return;
}
try
{
await Task.Delay(10 * 1000, token);
}
catch (TaskCanceledException)
{
// ignore
}
}
}
private Dictionary<int, Process> SnapshotProcesses() private Dictionary<int, Process> SnapshotProcesses()
{ {
Dictionary<int, Process> snapshot = new Dictionary<int, Process>(); Dictionary<int, Process> snapshot = new Dictionary<int, Process>();

View File

@@ -5,21 +5,13 @@ using GitHub.Services.Common;
using GitHub.Services.WebApi; using GitHub.Services.WebApi;
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Globalization;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using System.Net.Http; using System.Net.Http;
using System.Text;
using System.IO.Compression;
using System.Diagnostics;
using Newtonsoft.Json.Linq;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
using GitHub.Runner.Common; using GitHub.Runner.Common;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;
using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.ObjectTemplating;
namespace GitHub.Runner.Worker namespace GitHub.Runner.Worker
{ {
@@ -107,7 +99,7 @@ namespace GitHub.Runner.Worker
return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed); return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed);
} }
if (jobContext.WriteDebug) if (jobContext.Global.WriteDebug)
{ {
jobContext.SetRunnerContext("debug", "1"); jobContext.SetRunnerContext("debug", "1");
} }
@@ -122,13 +114,6 @@ namespace GitHub.Runner.Worker
_tempDirectoryManager = HostContext.GetService<ITempDirectoryManager>(); _tempDirectoryManager = HostContext.GetService<ITempDirectoryManager>();
_tempDirectoryManager.InitializeTempDirectory(jobContext); _tempDirectoryManager.InitializeTempDirectory(jobContext);
// // Expand container properties
// jobContext.Container?.ExpandProperties(jobContext.Variables);
// foreach (var sidecar in jobContext.SidecarContainers)
// {
// sidecar.ExpandProperties(jobContext.Variables);
// }
// Get the job extension. // Get the job extension.
Trace.Info("Getting job extension."); Trace.Info("Getting job extension.");
IJobExtension jobExtension = HostContext.CreateService<IJobExtension>(); IJobExtension jobExtension = HostContext.CreateService<IJobExtension>();
@@ -224,14 +209,14 @@ namespace GitHub.Runner.Worker
// Clean TEMP after finish process jobserverqueue, since there might be a pending fileupload still use the TEMP dir. // Clean TEMP after finish process jobserverqueue, since there might be a pending fileupload still use the TEMP dir.
_tempDirectoryManager?.CleanupTempDirectory(); _tempDirectoryManager?.CleanupTempDirectory();
if (!jobContext.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent)) if (!jobContext.Global.Features.HasFlag(PlanFeatures.JobCompletedPlanEvent))
{ {
Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}"); Trace.Info($"Skip raise job completed event call from worker because Plan version is {message.Plan.Version}");
return result; return result;
} }
Trace.Info("Raising job completed event."); Trace.Info("Raising job completed event.");
var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, jobContext.JobOutputs); var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, result, jobContext.JobOutputs, jobContext.ActionsEnvironment);
var completeJobRetryLimit = 5; var completeJobRetryLimit = 5;
var exceptions = new List<Exception>(); var exceptions = new List<Exception>();
@@ -254,6 +239,12 @@ namespace GitHub.Runner.Worker
Trace.Error(ex); Trace.Error(ex);
return TaskResult.Failed; return TaskResult.Failed;
} }
catch (TaskOrchestrationPlanTerminatedException ex)
{
Trace.Error($"TaskOrchestrationPlanTerminatedException received, while attempting to raise JobCompletedEvent for job {message.JobId}.");
Trace.Error(ex);
return TaskResult.Failed;
}
catch (Exception ex) catch (Exception ex)
{ {
Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}."); Trace.Error($"Catch exception while attempting to raise JobCompletedEvent for job {message.JobId}, job request {message.RequestId}.");

View File

@@ -100,12 +100,12 @@ namespace GitHub.Runner.Worker
RunnerActionPluginExecutionContext pluginContext = new RunnerActionPluginExecutionContext RunnerActionPluginExecutionContext pluginContext = new RunnerActionPluginExecutionContext
{ {
Inputs = inputs, Inputs = inputs,
Endpoints = context.Endpoints, Endpoints = context.Global.Endpoints,
Context = context.ExpressionValues Context = context.ExpressionValues
}; };
// variables // variables
foreach (var variable in context.Variables.AllVariables) foreach (var variable in context.Global.Variables.AllVariables)
{ {
pluginContext.Variables[variable.Name] = new VariableValue(variable.Value, variable.Secret); pluginContext.Variables[variable.Name] = new VariableValue(variable.Value, variable.Secret);
} }

View File

@@ -15,6 +15,14 @@ namespace GitHub.Runner.Worker
private static readonly Regex _propertyRegex = new Regex("^[a-zA-Z_][a-zA-Z0-9_]*$", RegexOptions.Compiled); private static readonly Regex _propertyRegex = new Regex("^[a-zA-Z_][a-zA-Z0-9_]*$", RegexOptions.Compiled);
private readonly DictionaryContextData _contextData = new DictionaryContextData(); private readonly DictionaryContextData _contextData = new DictionaryContextData();
public void ClearScope(string scopeName)
{
if (_contextData.TryGetValue(scopeName, out _))
{
_contextData[scopeName] = new DictionaryContextData();
}
}
public DictionaryContextData GetScope(string scopeName) public DictionaryContextData GetScope(string scopeName)
{ {
if (scopeName == null) if (scopeName == null)

View File

@@ -66,11 +66,11 @@ namespace GitHub.Runner.Worker
} }
var step = jobContext.JobSteps.Dequeue(); var step = jobContext.JobSteps.Dequeue();
var nextStep = jobContext.JobSteps.Count > 0 ? jobContext.JobSteps.Peek() : null;
Trace.Info($"Processing step: DisplayName='{step.DisplayName}'"); Trace.Info($"Processing step: DisplayName='{step.DisplayName}'");
ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext)); ArgUtil.NotNull(step.ExecutionContext, nameof(step.ExecutionContext));
ArgUtil.NotNull(step.ExecutionContext.Variables, nameof(step.ExecutionContext.Variables)); ArgUtil.NotNull(step.ExecutionContext.Global, nameof(step.ExecutionContext.Global));
ArgUtil.NotNull(step.ExecutionContext.Global.Variables, nameof(step.ExecutionContext.Global.Variables));
// Start // Start
step.ExecutionContext.Start(); step.ExecutionContext.Start();
@@ -82,27 +82,32 @@ namespace GitHub.Runner.Worker
step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo<SuccessFunction>(PipelineTemplateConstants.Success, 0, 0)); step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo<SuccessFunction>(PipelineTemplateConstants.Success, 0, 0));
step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo<HashFilesFunction>(PipelineTemplateConstants.HashFiles, 1, byte.MaxValue)); step.ExecutionContext.ExpressionFunctions.Add(new FunctionInfo<HashFilesFunction>(PipelineTemplateConstants.HashFiles, 1, byte.MaxValue));
// Initialize scope step.ExecutionContext.ExpressionValues["steps"] = step.ExecutionContext.Global.StepsContext.GetScope(step.ExecutionContext.ScopeName);
if (InitializeScope(step, scopeInputs))
{ // Populate env context for each step
// Populate env context for each step Trace.Info("Initialize Env context for step");
Trace.Info("Initialize Env context for step");
#if OS_WINDOWS #if OS_WINDOWS
var envContext = new DictionaryContextData(); var envContext = new DictionaryContextData();
#else #else
var envContext = new CaseSensitiveDictionaryContextData(); var envContext = new CaseSensitiveDictionaryContextData();
#endif #endif
step.ExecutionContext.ExpressionValues["env"] = envContext;
foreach (var pair in step.ExecutionContext.EnvironmentVariables)
{
envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
if (step is IActionRunner actionStep) // Global env
{ foreach (var pair in step.ExecutionContext.Global.EnvironmentVariables)
// Set GITHUB_ACTION {
step.ExecutionContext.SetGitHubContext("action", actionStep.Action.Name); envContext[pair.Key] = new StringContextData(pair.Value ?? string.Empty);
}
step.ExecutionContext.ExpressionValues["env"] = envContext;
bool evaluateStepEnvFailed = false;
if (step is IActionRunner actionStep)
{
// Set GITHUB_ACTION
step.ExecutionContext.SetGitHubContext("action", actionStep.Action.Name);
try
{
// Evaluate and merge action's env block to env context // Evaluate and merge action's env block to env context
var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(); var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator();
var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer); var actionEnvironment = templateEvaluator.EvaluateStepEnvironment(actionStep.Action.Environment, step.ExecutionContext.ExpressionValues, step.ExecutionContext.ExpressionFunctions, VarUtil.EnvironmentVariableKeyComparer);
@@ -111,7 +116,18 @@ namespace GitHub.Runner.Worker
envContext[env.Key] = new StringContextData(env.Value ?? string.Empty); envContext[env.Key] = new StringContextData(env.Value ?? string.Empty);
} }
} }
catch (Exception ex)
{
// fail the step since there is an evaluate error.
Trace.Info("Caught exception from expression for step.env");
evaluateStepEnvFailed = true;
step.ExecutionContext.Error(ex);
CompleteStep(step, TaskResult.Failed);
}
}
if (!evaluateStepEnvFailed)
{
try try
{ {
// Register job cancellation call back only if job cancellation token not been fire before each step run // Register job cancellation call back only if job cancellation token not been fire before each step run
@@ -195,19 +211,19 @@ namespace GitHub.Runner.Worker
{ {
// Condition == false // Condition == false
Trace.Info("Skipping step due to condition evaluation."); Trace.Info("Skipping step due to condition evaluation.");
CompleteStep(step, nextStep, TaskResult.Skipped, resultCode: conditionTraceWriter.Trace); CompleteStep(step, TaskResult.Skipped, resultCode: conditionTraceWriter.Trace);
} }
else if (conditionEvaluateError != null) else if (conditionEvaluateError != null)
{ {
// fail the step since there is an evaluate error. // fail the step since there is an evaluate error.
step.ExecutionContext.Error(conditionEvaluateError); step.ExecutionContext.Error(conditionEvaluateError);
CompleteStep(step, nextStep, TaskResult.Failed); CompleteStep(step, TaskResult.Failed);
} }
else else
{ {
// Run the step. // Run the step.
await RunStepAsync(step, jobContext.CancellationToken); await RunStepAsync(step, jobContext.CancellationToken);
CompleteStep(step, nextStep); CompleteStep(step);
} }
} }
finally finally
@@ -270,40 +286,7 @@ namespace GitHub.Runner.Worker
step.ExecutionContext.SetTimeout(timeout); step.ExecutionContext.SetTimeout(timeout);
} }
#if OS_WINDOWS await EncodingUtil.SetEncoding(HostContext, Trace, step.ExecutionContext.CancellationToken);
try
{
if (Console.InputEncoding.CodePage != 65001)
{
using (var p = HostContext.CreateService<IProcessInvoker>())
{
// Use UTF8 code page
int exitCode = await p.ExecuteAsync(workingDirectory: HostContext.GetDirectory(WellKnownDirectory.Work),
fileName: WhichUtil.Which("chcp", true, Trace),
arguments: "65001",
environment: null,
requireExitCodeZero: false,
outputEncoding: null,
killProcessOnCancel: false,
redirectStandardIn: null,
inheritConsoleHandler: true,
cancellationToken: step.ExecutionContext.CancellationToken);
if (exitCode == 0)
{
Trace.Info("Successfully returned to code page 65001 (UTF8)");
}
else
{
Trace.Warning($"'chcp 65001' failed with exit code {exitCode}");
}
}
}
}
catch (Exception ex)
{
Trace.Warning($"'chcp 65001' failed with exception {ex.Message}");
}
#endif
try try
{ {
@@ -369,117 +352,9 @@ namespace GitHub.Runner.Worker
step.ExecutionContext.Debug($"Finishing: {step.DisplayName}"); step.ExecutionContext.Debug($"Finishing: {step.DisplayName}");
} }
private bool InitializeScope(IStep step, Dictionary<string, PipelineContextData> scopeInputs) private void CompleteStep(IStep step, TaskResult? result = null, string resultCode = null)
{ {
var executionContext = step.ExecutionContext; var executionContext = step.ExecutionContext;
var stepsContext = executionContext.StepsContext;
if (!string.IsNullOrEmpty(executionContext.ScopeName))
{
// Gather uninitialized current and ancestor scopes
var scope = executionContext.Scopes[executionContext.ScopeName];
var scopesToInitialize = default(Stack<ContextScope>);
while (scope != null && !scopeInputs.ContainsKey(scope.Name))
{
if (scopesToInitialize == null)
{
scopesToInitialize = new Stack<ContextScope>();
}
scopesToInitialize.Push(scope);
scope = string.IsNullOrEmpty(scope.ParentName) ? null : executionContext.Scopes[scope.ParentName];
}
// Initialize current and ancestor scopes
while (scopesToInitialize?.Count > 0)
{
scope = scopesToInitialize.Pop();
executionContext.Debug($"Initializing scope '{scope.Name}'");
executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.ParentName);
executionContext.ExpressionValues["inputs"] = !String.IsNullOrEmpty(scope.ParentName) ? scopeInputs[scope.ParentName] : null;
var templateEvaluator = executionContext.ToPipelineTemplateEvaluator();
var inputs = default(DictionaryContextData);
try
{
inputs = templateEvaluator.EvaluateStepScopeInputs(scope.Inputs, executionContext.ExpressionValues, executionContext.ExpressionFunctions);
}
catch (Exception ex)
{
Trace.Info($"Caught exception from initialize scope '{scope.Name}'");
Trace.Error(ex);
executionContext.Error(ex);
executionContext.Complete(TaskResult.Failed);
return false;
}
scopeInputs[scope.Name] = inputs;
}
}
// Setup expression values
var scopeName = executionContext.ScopeName;
executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scopeName);
executionContext.ExpressionValues["inputs"] = string.IsNullOrEmpty(scopeName) ? null : scopeInputs[scopeName];
return true;
}
private void CompleteStep(IStep step, IStep nextStep, TaskResult? result = null, string resultCode = null)
{
var executionContext = step.ExecutionContext;
if (!string.IsNullOrEmpty(executionContext.ScopeName))
{
// Gather current and ancestor scopes to finalize
var scope = executionContext.Scopes[executionContext.ScopeName];
var scopesToFinalize = default(Queue<ContextScope>);
var nextStepScopeName = nextStep?.ExecutionContext.ScopeName;
while (scope != null &&
!string.Equals(nextStepScopeName, scope.Name, StringComparison.OrdinalIgnoreCase) &&
!(nextStepScopeName ?? string.Empty).StartsWith($"{scope.Name}.", StringComparison.OrdinalIgnoreCase))
{
if (scopesToFinalize == null)
{
scopesToFinalize = new Queue<ContextScope>();
}
scopesToFinalize.Enqueue(scope);
scope = string.IsNullOrEmpty(scope.ParentName) ? null : executionContext.Scopes[scope.ParentName];
}
// Finalize current and ancestor scopes
var stepsContext = step.ExecutionContext.StepsContext;
while (scopesToFinalize?.Count > 0)
{
scope = scopesToFinalize.Dequeue();
executionContext.Debug($"Finalizing scope '{scope.Name}'");
executionContext.ExpressionValues["steps"] = stepsContext.GetScope(scope.Name);
executionContext.ExpressionValues["inputs"] = null;
var templateEvaluator = executionContext.ToPipelineTemplateEvaluator();
var outputs = default(DictionaryContextData);
try
{
outputs = templateEvaluator.EvaluateStepScopeOutputs(scope.Outputs, executionContext.ExpressionValues, executionContext.ExpressionFunctions);
}
catch (Exception ex)
{
Trace.Info($"Caught exception from finalize scope '{scope.Name}'");
Trace.Error(ex);
executionContext.Error(ex);
executionContext.Complete(TaskResult.Failed);
return;
}
if (outputs?.Count > 0)
{
var parentScopeName = scope.ParentName;
var contextName = scope.ContextName;
foreach (var pair in outputs)
{
var outputName = pair.Key;
var outputValue = pair.Value.ToString();
stepsContext.SetOutput(parentScopeName, contextName, outputName, outputValue, out var reference);
executionContext.Debug($"{reference}='{outputValue}'");
}
}
}
}
executionContext.Complete(result, resultCode: resultCode); executionContext.Complete(result, resultCode: resultCode);
} }

View File

@@ -7,7 +7,8 @@
"name": "string", "name": "string",
"description": "string", "description": "string",
"inputs": "inputs", "inputs": "inputs",
"runs": "runs" "runs": "runs",
"outputs": "outputs"
}, },
"loose-key-type": "non-empty-string", "loose-key-type": "non-empty-string",
"loose-value-type": "any" "loose-value-type": "any"
@@ -28,11 +29,26 @@
"loose-value-type": "any" "loose-value-type": "any"
} }
}, },
"outputs": {
"mapping": {
"loose-key-type": "non-empty-string",
"loose-value-type": "outputs-attributes"
}
},
"outputs-attributes": {
"mapping": {
"properties": {
"description": "string",
"value": "output-value"
}
}
},
"runs": { "runs": {
"one-of": [ "one-of": [
"container-runs", "container-runs",
"node12-runs", "node12-runs",
"plugin-runs" "plugin-runs",
"composite-runs"
] ]
}, },
"container-runs": { "container-runs": {
@@ -83,12 +99,56 @@
} }
} }
}, },
"composite-runs": {
"mapping": {
"properties": {
"using": "non-empty-string",
"steps": "composite-steps"
}
}
},
"composite-steps": {
"sequence": {
"item-type": "composite-step"
}
},
"composite-step": {
"mapping": {
"properties": {
"name": "string-steps-context",
"id": "non-empty-string",
"run": {
"type": "string-steps-context",
"required": true
},
"env": "step-env",
"working-directory": "string-steps-context",
"shell": {
"type": "non-empty-string",
"required": true
}
}
}
},
"container-runs-context": { "container-runs-context": {
"context": [ "context": [
"inputs" "inputs"
], ],
"string": {} "string": {}
}, },
"output-value": {
"context": [
"github",
"strategy",
"matrix",
"steps",
"inputs",
"job",
"runner",
"env"
],
"string": {}
},
"input-default-context": { "input-default-context": {
"context": [ "context": [
"github", "github",
@@ -104,6 +164,37 @@
"string": { "string": {
"require-non-empty": true "require-non-empty": true
} }
},
"string-steps-context": {
"context": [
"github",
"inputs",
"strategy",
"matrix",
"steps",
"job",
"runner",
"env",
"hashFiles(1,255)"
],
"string": {}
},
"step-env": {
"context": [
"github",
"inputs",
"strategy",
"matrix",
"steps",
"job",
"runner",
"env",
"hashFiles(1,255)"
],
"mapping": {
"loose-key-type": "non-empty-string",
"loose-value-type": "string"
}
} }
} }
} }

View File

@@ -317,5 +317,37 @@ namespace GitHub.DistributedTask.WebApi
userState: userState, userState: userState,
cancellationToken: cancellationToken); cancellationToken: cancellationToken);
} }
/// <summary>
/// [Preview API] Resolves information required to download actions (URL, token) defined in an orchestration.
/// </summary>
/// <param name="scopeIdentifier">The project GUID to scope the request</param>
/// <param name="hubName">The name of the server hub: "build" for the Build server or "rm" for the Release Management server</param>
/// <param name="planId"></param>
/// <param name="actionReferenceList"></param>
/// <param name="userState"></param>
/// <param name="cancellationToken">The cancellation token to cancel operation.</param>
public virtual Task<ActionDownloadInfoCollection> ResolveActionDownloadInfoAsync(
Guid scopeIdentifier,
string hubName,
Guid planId,
ActionReferenceList actionReferenceList,
object userState = null,
CancellationToken cancellationToken = default)
{
HttpMethod httpMethod = new HttpMethod("POST");
Guid locationId = new Guid("27d7f831-88c1-4719-8ca1-6a061dad90eb");
object routeValues = new { scopeIdentifier = scopeIdentifier, hubName = hubName, planId = planId };
HttpContent content = new ObjectContent<ActionReferenceList>(actionReferenceList, new VssJsonMediaTypeFormatter(true));
return SendAsync<ActionDownloadInfoCollection>(
httpMethod,
locationId,
routeValues: routeValues,
version: new ApiResourceVersion(6.0, 1),
userState: userState,
cancellationToken: cancellationToken,
content: content);
}
} }
} }

View File

@@ -52,6 +52,7 @@ namespace GitHub.DistributedTask.ObjectTemplating
internal const String String = "string"; internal const String String = "string";
internal const String StringDefinition = "string-definition"; internal const String StringDefinition = "string-definition";
internal const String StringDefinitionProperties = "string-definition-properties"; internal const String StringDefinitionProperties = "string-definition-properties";
internal const String StringRunnerContextNoSecrets = "string-runner-context-no-secrets";
internal const String Structure = "structure"; internal const String Structure = "structure";
internal const String TemplateSchema = "template-schema"; internal const String TemplateSchema = "template-schema";
internal const String True = "true"; internal const String True = "true";

View File

@@ -24,7 +24,6 @@ namespace GitHub.DistributedTask.Pipelines
Environment = actionToClone.Environment?.Clone(); Environment = actionToClone.Environment?.Clone();
Inputs = actionToClone.Inputs?.Clone(); Inputs = actionToClone.Inputs?.Clone();
ContextName = actionToClone?.ContextName; ContextName = actionToClone?.ContextName;
ScopeName = actionToClone?.ScopeName;
DisplayNameToken = actionToClone.DisplayNameToken?.Clone(); DisplayNameToken = actionToClone.DisplayNameToken?.Clone();
} }
@@ -41,9 +40,6 @@ namespace GitHub.DistributedTask.Pipelines
[DataMember(EmitDefaultValue = false)] [DataMember(EmitDefaultValue = false)]
public TemplateToken DisplayNameToken { get; set; } public TemplateToken DisplayNameToken { get; set; }
[DataMember(EmitDefaultValue = false)]
public String ScopeName { get; set; }
[DataMember(EmitDefaultValue = false)] [DataMember(EmitDefaultValue = false)]
public String ContextName { get; set; } public String ContextName { get; set; }

View File

@@ -39,10 +39,10 @@ namespace GitHub.DistributedTask.Pipelines
DictionaryContextData contextData, DictionaryContextData contextData,
WorkspaceOptions workspaceOptions, WorkspaceOptions workspaceOptions,
IEnumerable<JobStep> steps, IEnumerable<JobStep> steps,
IEnumerable<ContextScope> scopes,
IList<String> fileTable, IList<String> fileTable,
TemplateToken jobOutputs, TemplateToken jobOutputs,
IList<TemplateToken> defaults) IList<TemplateToken> defaults,
ActionsEnvironmentReference actionsEnvironment)
{ {
this.MessageType = JobRequestMessageTypes.PipelineAgentJobRequest; this.MessageType = JobRequestMessageTypes.PipelineAgentJobRequest;
this.Plan = plan; this.Plan = plan;
@@ -55,16 +55,11 @@ namespace GitHub.DistributedTask.Pipelines
this.Resources = jobResources; this.Resources = jobResources;
this.Workspace = workspaceOptions; this.Workspace = workspaceOptions;
this.JobOutputs = jobOutputs; this.JobOutputs = jobOutputs;
this.ActionsEnvironment = actionsEnvironment;
m_variables = new Dictionary<String, VariableValue>(variables, StringComparer.OrdinalIgnoreCase); m_variables = new Dictionary<String, VariableValue>(variables, StringComparer.OrdinalIgnoreCase);
m_maskHints = new List<MaskHint>(maskHints); m_maskHints = new List<MaskHint>(maskHints);
m_steps = new List<JobStep>(steps); m_steps = new List<JobStep>(steps);
if (scopes != null)
{
m_scopes = new List<ContextScope>(scopes);
}
if (environmentVariables?.Count > 0) if (environmentVariables?.Count > 0)
{ {
m_environmentVariables = new List<TemplateToken>(environmentVariables); m_environmentVariables = new List<TemplateToken>(environmentVariables);
@@ -234,6 +229,13 @@ namespace GitHub.DistributedTask.Pipelines
} }
} }
[DataMember(EmitDefaultValue = false)]
public ActionsEnvironmentReference ActionsEnvironment
{
get;
set;
}
/// <summary> /// <summary>
/// Gets the collection of variables associated with the current context. /// Gets the collection of variables associated with the current context.
/// </summary> /// </summary>
@@ -261,18 +263,6 @@ namespace GitHub.DistributedTask.Pipelines
} }
} }
public IList<ContextScope> Scopes
{
get
{
if (m_scopes == null)
{
m_scopes = new List<ContextScope>();
}
return m_scopes;
}
}
/// <summary> /// <summary>
/// Gets the table of files used when parsing the pipeline (e.g. yaml files) /// Gets the table of files used when parsing the pipeline (e.g. yaml files)
/// </summary> /// </summary>
@@ -415,11 +405,6 @@ namespace GitHub.DistributedTask.Pipelines
m_maskHints = new List<MaskHint>(this.m_maskHints.Distinct()); m_maskHints = new List<MaskHint>(this.m_maskHints.Distinct());
} }
if (m_scopes?.Count == 0)
{
m_scopes = null;
}
if (m_variables?.Count == 0) if (m_variables?.Count == 0)
{ {
m_variables = null; m_variables = null;
@@ -447,9 +432,6 @@ namespace GitHub.DistributedTask.Pipelines
[DataMember(Name = "Steps", EmitDefaultValue = false)] [DataMember(Name = "Steps", EmitDefaultValue = false)]
private List<JobStep> m_steps; private List<JobStep> m_steps;
[DataMember(Name = "Scopes", EmitDefaultValue = false)]
private List<ContextScope> m_scopes;
[DataMember(Name = "Variables", EmitDefaultValue = false)] [DataMember(Name = "Variables", EmitDefaultValue = false)]
private IDictionary<String, VariableValue> m_variables; private IDictionary<String, VariableValue> m_variables;

View File

@@ -42,7 +42,12 @@ namespace GitHub.DistributedTask.Pipelines.ContextData
var floored = Math.Floor(m_value); var floored = Math.Floor(m_value);
if (m_value == floored && m_value <= (Double)Int32.MaxValue && m_value >= (Double)Int32.MinValue) if (m_value == floored && m_value <= (Double)Int32.MaxValue && m_value >= (Double)Int32.MinValue)
{ {
Int32 flooredInt = (Int32)floored; var flooredInt = (Int32)floored;
return (JToken)flooredInt;
}
else if (m_value == floored && m_value <= (Double)Int64.MaxValue && m_value >= (Double)Int64.MinValue)
{
var flooredInt = (Int64)floored;
return (JToken)flooredInt; return (JToken)flooredInt;
} }
else else

View File

@@ -1,53 +0,0 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Runtime.Serialization;
using GitHub.DistributedTask.ObjectTemplating.Tokens;
using Newtonsoft.Json;
namespace GitHub.DistributedTask.Pipelines
{
[DataContract]
[EditorBrowsable(EditorBrowsableState.Never)]
public sealed class ContextScope
{
[DataMember(EmitDefaultValue = false)]
public String Name { get; set; }
[IgnoreDataMember]
public String ContextName
{
get
{
var index = Name.LastIndexOf('.');
if (index >= 0)
{
return Name.Substring(index + 1);
}
return Name;
}
}
[IgnoreDataMember]
public String ParentName
{
get
{
var index = Name.LastIndexOf('.');
if (index >= 0)
{
return Name.Substring(0, index);
}
return String.Empty;
}
}
[DataMember(EmitDefaultValue = false)]
public TemplateToken Inputs { get; set; }
[DataMember(EmitDefaultValue = false)]
public TemplateToken Outputs { get; set; }
}
}

Some files were not shown because too many files have changed in this diff Show More