Compare commits

...

6 Commits

Author SHA1 Message Date
John Wesley Walker III
290e9248bf Removed BOM from modified files as demanded by PR linter. 2023-05-02 16:46:21 +00:00
John Wesley Walker III
75ffe93f62 When inferring additional secrets from multi-line content, ignore very short lines. 2023-05-02 16:32:09 +00:00
John Wesley Walker III
afcca9bfa4 Allow public SecretMasker constructor to be called with a null IEnumerable<ValueEncoder> 2023-04-06 10:46:35 +02:00
John Wesley Walker III
157e03616e Set file encodings back to UTF8 (no-BOM) 2023-04-05 12:04:48 +00:00
John Wesley Walker III
30f686b9c2 Added a unit test for thorough Base64 secret masking. 2023-04-05 11:43:26 +00:00
John Wesley Walker III
ec5d72810f Omit Base64 padding characters from Base64 representations of secrets. 2023-04-05 08:27:23 +00:00
9 changed files with 241 additions and 168 deletions

View File

@@ -53,7 +53,7 @@ namespace GitHub.Runner.Common
private static int[] _vssHttpCredentialEventIds = new int[] { 11, 13, 14, 15, 16, 17, 18, 20, 21, 22, 27, 29 }; private static int[] _vssHttpCredentialEventIds = new int[] { 11, 13, 14, 15, 16, 17, 18, 20, 21, 22, 27, 29 };
private readonly ConcurrentDictionary<Type, object> _serviceInstances = new(); private readonly ConcurrentDictionary<Type, object> _serviceInstances = new();
private readonly ConcurrentDictionary<Type, Type> _serviceTypes = new(); private readonly ConcurrentDictionary<Type, Type> _serviceTypes = new();
private readonly ISecretMasker _secretMasker = new SecretMasker(); private readonly ISecretMasker _secretMasker;
private readonly List<ProductInfoHeaderValue> _userAgents = new() { new ProductInfoHeaderValue($"GitHubActionsRunner-{BuildConstants.RunnerPackage.PackageName}", BuildConstants.RunnerPackage.Version) }; private readonly List<ProductInfoHeaderValue> _userAgents = new() { new ProductInfoHeaderValue($"GitHubActionsRunner-{BuildConstants.RunnerPackage.PackageName}", BuildConstants.RunnerPackage.Version) };
private CancellationTokenSource _runnerShutdownTokenSource = new(); private CancellationTokenSource _runnerShutdownTokenSource = new();
private object _perfLock = new(); private object _perfLock = new();
@@ -82,17 +82,20 @@ namespace GitHub.Runner.Common
_loadContext = AssemblyLoadContext.GetLoadContext(typeof(HostContext).GetTypeInfo().Assembly); _loadContext = AssemblyLoadContext.GetLoadContext(typeof(HostContext).GetTypeInfo().Assembly);
_loadContext.Unloading += LoadContext_Unloading; _loadContext.Unloading += LoadContext_Unloading;
this.SecretMasker.AddValueEncoder(ValueEncoders.Base64StringEscape); var masks = new List<ValueEncoder>()
this.SecretMasker.AddValueEncoder(ValueEncoders.Base64StringEscapeShift1); {
this.SecretMasker.AddValueEncoder(ValueEncoders.Base64StringEscapeShift2); ValueEncoders.EnumerateBase64Variations,
this.SecretMasker.AddValueEncoder(ValueEncoders.CommandLineArgumentEscape); ValueEncoders.CommandLineArgumentEscape,
this.SecretMasker.AddValueEncoder(ValueEncoders.ExpressionStringEscape); ValueEncoders.ExpressionStringEscape,
this.SecretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape); ValueEncoders.JsonStringEscape,
this.SecretMasker.AddValueEncoder(ValueEncoders.UriDataEscape); ValueEncoders.UriDataEscape,
this.SecretMasker.AddValueEncoder(ValueEncoders.XmlDataEscape); ValueEncoders.XmlDataEscape,
this.SecretMasker.AddValueEncoder(ValueEncoders.TrimDoubleQuotes); ValueEncoders.TrimDoubleQuotes,
this.SecretMasker.AddValueEncoder(ValueEncoders.PowerShellPreAmpersandEscape); ValueEncoders.PowerShellPreAmpersandEscape,
this.SecretMasker.AddValueEncoder(ValueEncoders.PowerShellPostAmpersandEscape); ValueEncoders.PowerShellPostAmpersandEscape
};
_secretMasker = new SecretMasker(masks);
// Create StdoutTraceListener if ENV is set // Create StdoutTraceListener if ENV is set
StdoutTraceListener stdoutTraceListener = null; StdoutTraceListener stdoutTraceListener = null;

View File

@@ -1,4 +1,4 @@
using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.Pipelines.ContextData;
using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.WebApi;
using GitHub.Runner.Worker.Container; using GitHub.Runner.Worker.Container;
using System; using System;
@@ -405,8 +405,9 @@ namespace GitHub.Runner.Worker
Trace.Info($"Add new secret mask with length of {command.Data.Length}"); Trace.Info($"Add new secret mask with length of {command.Data.Length}");
// Also add each individual line. Typically individual lines are processed from STDOUT of child processes. // Also add each individual line. Typically individual lines are processed from STDOUT of child processes.
var split = command.Data.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); var auxiliarySecrets = command.Data.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
foreach (var item in split) .Where(candidate => candidate.Length >= HostContext.SecretMasker.DerivedSecretRecommendedMinimumLength);
foreach (var item in auxiliarySecrets)
{ {
HostContext.SecretMasker.AddValue(item); HostContext.SecretMasker.AddValue(item);
} }

View File

@@ -1,12 +1,12 @@
using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.WebApi;
using Pipelines = GitHub.DistributedTask.Pipelines; using Pipelines = GitHub.DistributedTask.Pipelines;
using GitHub.Runner.Common.Util; using GitHub.Runner.Common.Util;
using Newtonsoft.Json; using Newtonsoft.Json;
using System; using System;
using System.Linq;
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using GitHub.Services.WebApi;
using GitHub.Runner.Common; using GitHub.Runner.Common;
using GitHub.Runner.Sdk; using GitHub.Runner.Sdk;
using System.Text; using System.Text;
@@ -156,8 +156,9 @@ namespace GitHub.Runner.Worker
HostContext.SecretMasker.AddValue(value); HostContext.SecretMasker.AddValue(value);
// Also add each individual line. Typically individual lines are processed from STDOUT of child processes. // Also add each individual line. Typically individual lines are processed from STDOUT of child processes.
var split = value.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); var auxiliarySecrets = value.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
foreach (var item in split) .Where(candidate => candidate.Length >= HostContext.SecretMasker.DerivedSecretRecommendedMinimumLength);
foreach (var item in auxiliarySecrets)
{ {
HostContext.SecretMasker.AddValue(item); HostContext.SecretMasker.AddValue(item);
} }

View File

@@ -1,6 +1,6 @@
using System; using System;
using System.Collections.Generic;
using System.ComponentModel; using System.ComponentModel;
using System.Linq;
using GitHub.DistributedTask.Logging; using GitHub.DistributedTask.Logging;
namespace GitHub.DistributedTask.Expressions2.Sdk namespace GitHub.DistributedTask.Expressions2.Sdk
@@ -55,7 +55,7 @@ namespace GitHub.DistributedTask.Expressions2.Sdk
try try
{ {
// Evaluate // Evaluate
secretMasker = secretMasker?.Clone() ?? new SecretMasker(); secretMasker = secretMasker?.Clone() ?? new SecretMasker(Enumerable.Empty<ValueEncoder>());
trace = new EvaluationTraceWriter(trace, secretMasker); trace = new EvaluationTraceWriter(trace, secretMasker);
var context = new EvaluationContext(trace, secretMasker, state, options, this); var context = new EvaluationContext(trace, secretMasker, state, options, this);
trace.Info($"Evaluating: {ConvertToExpression()}"); trace.Info($"Evaluating: {ConvertToExpression()}");

View File

@@ -1,4 +1,4 @@
using System; using System;
using System.ComponentModel; using System.ComponentModel;
namespace GitHub.DistributedTask.Logging namespace GitHub.DistributedTask.Logging
@@ -6,9 +6,9 @@ namespace GitHub.DistributedTask.Logging
[EditorBrowsable(EditorBrowsableState.Never)] [EditorBrowsable(EditorBrowsableState.Never)]
public interface ISecretMasker public interface ISecretMasker
{ {
int DerivedSecretRecommendedMinimumLength { get; }
void AddRegex(String pattern); void AddRegex(String pattern);
void AddValue(String value); void AddValue(String value);
void AddValueEncoder(ValueEncoder encoder);
ISecretMasker Clone(); ISecretMasker Clone();
String MaskSecrets(String input); String MaskSecrets(String input);
} }

View File

@@ -1,4 +1,4 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.ComponentModel; using System.ComponentModel;
using System.Linq; using System.Linq;
@@ -10,11 +10,11 @@ namespace GitHub.DistributedTask.Logging
[EditorBrowsable(EditorBrowsableState.Never)] [EditorBrowsable(EditorBrowsableState.Never)]
public sealed class SecretMasker : ISecretMasker, IDisposable public sealed class SecretMasker : ISecretMasker, IDisposable
{ {
public SecretMasker() public SecretMasker(IEnumerable<ValueEncoder> encoders)
{ {
m_originalValueSecrets = new HashSet<ValueSecret>(); m_originalValueSecrets = new HashSet<ValueSecret>();
m_regexSecrets = new HashSet<RegexSecret>(); m_regexSecrets = new HashSet<RegexSecret>();
m_valueEncoders = new HashSet<ValueEncoder>(); m_valueEncoders = new HashSet<ValueEncoder>(encoders ?? Enumerable.Empty<ValueEncoder>());
m_valueSecrets = new HashSet<ValueSecret>(); m_valueSecrets = new HashSet<ValueSecret>();
} }
@@ -40,6 +40,19 @@ namespace GitHub.DistributedTask.Logging
} }
} }
/// <summary>
/// Provide callers with a recommendation on what to consider a secret.
/// This is helpful in cases where JSON (for example) is broken into multiple lines
/// and we don't want to start treating standalone JSON control characters as secrets.
/// </summary>
public int DerivedSecretRecommendedMinimumLength
{
get
{
return 3;
}
}
/// <summary> /// <summary>
/// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time. /// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time.
/// </summary> /// </summary>
@@ -104,15 +117,11 @@ namespace GitHub.DistributedTask.Logging
} }
} }
// Compute the encoded values. var secretVariations = valueEncoders.SelectMany(encoder => encoder(value))
foreach (ValueEncoder valueEncoder in valueEncoders) .Where(variation => !string.IsNullOrEmpty(variation))
{ .Distinct()
String encodedValue = valueEncoder(value); .Select(variation => new ValueSecret(variation));
if (!String.IsNullOrEmpty(encodedValue)) valueSecrets.AddRange(secretVariations);
{
valueSecrets.Add(new ValueSecret(encodedValue));
}
}
// Write section. // Write section.
try try
@@ -135,69 +144,6 @@ namespace GitHub.DistributedTask.Logging
} }
} }
/// <summary>
/// This implementation assumes no more than one thread is adding regexes, values, or encoders at any given time.
/// </summary>
public void AddValueEncoder(ValueEncoder encoder)
{
ValueSecret[] originalSecrets;
// Read section.
try
{
m_lock.EnterReadLock();
// Test whether already added.
if (m_valueEncoders.Contains(encoder))
{
return;
}
// Read the original value secrets.
originalSecrets = m_originalValueSecrets.ToArray();
}
finally
{
if (m_lock.IsReadLockHeld)
{
m_lock.ExitReadLock();
}
}
// Compute the encoded values.
var encodedSecrets = new List<ValueSecret>();
foreach (ValueSecret originalSecret in originalSecrets)
{
String encodedValue = encoder(originalSecret.m_value);
if (!String.IsNullOrEmpty(encodedValue))
{
encodedSecrets.Add(new ValueSecret(encodedValue));
}
}
// Write section.
try
{
m_lock.EnterWriteLock();
// Add the encoder.
m_valueEncoders.Add(encoder);
// Add the values.
foreach (ValueSecret encodedSecret in encodedSecrets)
{
m_valueSecrets.Add(encodedSecret);
}
}
finally
{
if (m_lock.IsWriteLockHeld)
{
m_lock.ExitWriteLock();
}
}
}
public ISecretMasker Clone() => new SecretMasker(this); public ISecretMasker Clone() => new SecretMasker(this);
public void Dispose() public void Dispose()

View File

@@ -1,73 +1,97 @@
using System; using System;
using System.Collections.Generic;
using System.ComponentModel; using System.ComponentModel;
using System.Linq;
using System.Security; using System.Security;
using System.Text; using System.Text;
using System.Text.RegularExpressions;
using Newtonsoft.Json; using Newtonsoft.Json;
namespace GitHub.DistributedTask.Logging namespace GitHub.DistributedTask.Logging
{ {
[EditorBrowsable(EditorBrowsableState.Never)] [EditorBrowsable(EditorBrowsableState.Never)]
public delegate String ValueEncoder(String value); public delegate IEnumerable<string> ValueEncoder(string value);
[EditorBrowsable(EditorBrowsableState.Never)] [EditorBrowsable(EditorBrowsableState.Never)]
public static class ValueEncoders public static class ValueEncoders
{ {
public static String Base64StringEscape(String value)
{
return Convert.ToBase64String(Encoding.UTF8.GetBytes(value));
}
// Base64 is 6 bits -> char public static IEnumerable<string> EnumerateBase64Variations(string value)
// A byte is 8 bits
// When end user doing somthing like base64(user:password)
// The length of the leading content will cause different base64 encoding result on the password
// So we add base64(value shifted 1 and two bytes) as secret as well.
// B1 B2 B3 B4 B5 B6 B7
// 000000|00 0000|0000 00|000000| 000000|00 0000|0000 00|000000|
// Char1 Char2 Char3 Char4
// See the above, the first byte has a character beginning at index 0, the second byte has a character beginning at index 4, the third byte has a character beginning at index 2 and then the pattern repeats
// We register byte offsets for all these possible values
public static String Base64StringEscapeShift1(String value)
{ {
return Base64StringEscapeShift(value, 1); if (!string.IsNullOrEmpty(value))
} {
// A byte is 8 bits. A Base64 "digit" can hold a maximum of 6 bits (2^64 - 1, or values 0 to 63).
// As a result, many Unicode characters (including single-byte letters) cannot be represented using a single Base64 digit.
// Furthermore, on average a Base64 string will be about 33% longer than the original text.
// This is because it generally requires 4 Base64 digits to represent 3 Unicode bytes. (4 / 3 ~ 1.33)
//
// Because of this 4:3 ratio (or, more precisely, 8 bits : 6 bits ratio), there's a cyclical pattern
// to when a byte boundary aligns with a Base64 digit boundary.
// The pattern repeats every 24 bits (the lowest common multiple of 8 and 6).
//
// |-----------24 bits-------------|-----------24 bits------------|
// Base64 Digits: |digit 0|digit 1|digit 2|digit 3|digit 4|digit 5|digit 6|digit7|
// Allocated Bits: aaaaaa aaBBBB BBBBcc cccccc DDDDDD DDeeee eeeeFF FFFFFF
// Unicode chars: |0th char |1st char |2nd char |3rd char |4th char |5th char |
public static String Base64StringEscapeShift2(String value) // Depending on alignment, the Base64-encoded secret can take any of 3 basic forms.
{ // For example, the Base64 digits representing "abc" could appear as any of the following:
return Base64StringEscapeShift(value, 2); // "YWJj" when aligned
// ".!FiYw==" when preceded by 3x + 1 bytes
// "..!hYmM=" when preceded by 3x + 2 bytes
// (where . represents an unrelated Base64 digit, ! represents a Base64 digit that should be masked, and x represents any non-negative integer)
var rawBytes = Encoding.UTF8.GetBytes(value);
for (var offset = 0; offset <= 2; offset++)
{
var prunedBytes = rawBytes.Skip(offset).ToArray();
if (prunedBytes.Length > 0)
{
// Don't include Base64 padding characters (=) in Base64 representations of the secret.
// They don't represent anything interesting, so they don't need to be masked.
// (Some clients omit the padding, so we want to be sure we recognize the secret regardless of whether the padding is present or not.)
var buffer = new StringBuilder(Convert.ToBase64String(prunedBytes).TrimEnd(BASE64_PADDING_SUFFIX));
yield return buffer.ToString();
// Also, yield the RFC4648-equivalent RegEx.
buffer.Replace('+', '-');
buffer.Replace('/', '_');
yield return buffer.ToString();
}
}
}
} }
// Used when we pass environment variables to docker to escape " with \" // Used when we pass environment variables to docker to escape " with \"
public static String CommandLineArgumentEscape(String value) public static IEnumerable<string> CommandLineArgumentEscape(string value)
{ {
return value.Replace("\"", "\\\""); yield return value.Replace("\"", "\\\"");
} }
public static String ExpressionStringEscape(String value) public static IEnumerable<string> ExpressionStringEscape(string value)
{ {
return Expressions2.Sdk.ExpressionUtility.StringEscape(value); yield return Expressions2.Sdk.ExpressionUtility.StringEscape(value);
} }
public static String JsonStringEscape(String value) public static IEnumerable<string> JsonStringEscape(string value)
{ {
// Convert to a JSON string and then remove the leading/trailing double-quote. // Convert to a JSON string and then remove the leading/trailing double-quote.
String jsonString = JsonConvert.ToString(value); String jsonString = JsonConvert.ToString(value);
String jsonEscapedValue = jsonString.Substring(startIndex: 1, length: jsonString.Length - 2); String jsonEscapedValue = jsonString.Substring(startIndex: 1, length: jsonString.Length - 2);
return jsonEscapedValue; yield return jsonEscapedValue;
} }
public static String UriDataEscape(String value) public static IEnumerable<string> UriDataEscape(string value)
{ {
return UriDataEscape(value, 65519); yield return UriDataEscape(value, 65519);
} }
public static String XmlDataEscape(String value) public static IEnumerable<string> XmlDataEscape(string value)
{ {
return SecurityElement.Escape(value); yield return SecurityElement.Escape(value);
} }
public static String TrimDoubleQuotes(String value) public static IEnumerable<string> TrimDoubleQuotes(string value)
{ {
var trimmed = string.Empty; var trimmed = string.Empty;
if (!string.IsNullOrEmpty(value) && if (!string.IsNullOrEmpty(value) &&
@@ -78,17 +102,17 @@ namespace GitHub.DistributedTask.Logging
trimmed = value.Substring(1, value.Length - 2); trimmed = value.Substring(1, value.Length - 2);
} }
return trimmed; yield return trimmed;
} }
public static String PowerShellPreAmpersandEscape(String value) public static IEnumerable<string> PowerShellPreAmpersandEscape(string value)
{ {
// if the secret is passed to PS as a command and it causes an error, sections of it can be surrounded by color codes // if the secret is passed to PS as a command and it causes an error, sections of it can be surrounded by color codes
// or printed individually. // or printed individually.
// The secret secretpart1&secretpart2&secretpart3 would be split into 2 sections: // The secret secretpart1&secretpart2&secretpart3 would be split into 2 sections:
// 'secretpart1&secretpart2&' and 'secretpart3'. This method masks for the first section. // 'secretpart1&secretpart2&' and 'secretpart3'. This method masks for the first section.
// The secret secretpart1&+secretpart2&secretpart3 would be split into 2 sections: // The secret secretpart1&+secretpart2&secretpart3 would be split into 2 sections:
// 'secretpart1&+' and (no 's') 'ecretpart2&secretpart3'. This method masks for the first section. // 'secretpart1&+' and (no 's') 'ecretpart2&secretpart3'. This method masks for the first section.
@@ -112,10 +136,10 @@ namespace GitHub.DistributedTask.Logging
} }
} }
return trimmed; yield return trimmed;
} }
public static String PowerShellPostAmpersandEscape(String value) public static IEnumerable<string> PowerShellPostAmpersandEscape(string value)
{ {
var trimmed = string.Empty; var trimmed = string.Empty;
if (!string.IsNullOrEmpty(value) && value.Contains("&")) if (!string.IsNullOrEmpty(value) && value.Contains("&"))
@@ -137,27 +161,10 @@ namespace GitHub.DistributedTask.Logging
} }
} }
return trimmed; yield return trimmed;
} }
private static string Base64StringEscapeShift(String value, int shift) private static string UriDataEscape(string value, Int32 maxSegmentSize)
{
var bytes = Encoding.UTF8.GetBytes(value);
if (bytes.Length > shift)
{
var shiftArray = new byte[bytes.Length - shift];
Array.Copy(bytes, shift, shiftArray, 0, bytes.Length - shift);
return Convert.ToBase64String(shiftArray);
}
else
{
return Convert.ToBase64String(bytes);
}
}
private static String UriDataEscape(
String value,
Int32 maxSegmentSize)
{ {
if (value.Length <= maxSegmentSize) if (value.Length <= maxSegmentSize)
{ {
@@ -183,5 +190,7 @@ namespace GitHub.DistributedTask.Logging
return result.ToString(); return result.ToString();
} }
private const char BASE64_PADDING_SUFFIX = '=';
} }
} }

View File

@@ -1,4 +1,3 @@
using GitHub.Runner.Common.Util;
using System; using System;
using System.IO; using System.IO;
using System.Reflection; using System.Reflection;
@@ -13,6 +12,7 @@ namespace GitHub.Runner.Common.Tests
{ {
private HostContext _hc; private HostContext _hc;
private CancellationTokenSource _tokenSource; private CancellationTokenSource _tokenSource;
private const string EXPECTED_SECRET_MASK = "***";
[Fact] [Fact]
[Trait("Level", "L0")] [Trait("Level", "L0")]
@@ -95,11 +95,11 @@ namespace GitHub.Runner.Common.Tests
Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123Pass%20word%20123%21123")); Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123Pass%20word%20123%21123"));
Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123Pass&lt;word&gt;123!123")); Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123Pass&lt;word&gt;123!123"));
Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123Pass''word''123!123")); Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123Pass''word''123!123"));
Assert.Equal("OlBh***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($":Password123!")))); Assert.Equal("OlBh***==", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($":Password123!"))));
Assert.Equal("YTpQ***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"a:Password123!")))); Assert.Equal("YTpQ***=", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"a:Password123!"))));
Assert.Equal("YWI6***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"ab:Password123!")))); Assert.Equal("YWI6***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"ab:Password123!"))));
Assert.Equal("YWJjOlBh***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"abc:Password123!")))); Assert.Equal("YWJjOlBh***==", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"abc:Password123!"))));
Assert.Equal("YWJjZDpQ***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"abcd:Password123!")))); Assert.Equal("YWJjZDpQ***=", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"abcd:Password123!"))));
Assert.Equal("YWJjZGU6***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"abcde:Password123!")))); Assert.Equal("YWJjZGU6***", _hc.SecretMasker.MaskSecrets(Convert.ToBase64String(Encoding.UTF8.GetBytes($"abcde:Password123!"))));
Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123Password123!!123")); Assert.Equal("123***123", _hc.SecretMasker.MaskSecrets("123Password123!!123"));
Assert.Equal("123short123", _hc.SecretMasker.MaskSecrets("123short123")); Assert.Equal("123short123", _hc.SecretMasker.MaskSecrets("123short123"));
@@ -112,6 +112,116 @@ namespace GitHub.Runner.Common.Tests
} }
} }
[Fact]
[Trait("Level", "L0")]
[Trait("Category", "Common")]
public void Base64SecretMaskers()
{
// The following are good candidate strings for Base64 encoding because they include
// both standard and RFC 4648 Base64 digits in all offset variations.
// TeLL? noboDy~ SEcreT?
// tElL~ NEVER~ neveR?
// TIGht? Tight~ guard~
// pRIVAte~ guARd? TIghT~
// KeeP~ TIgHT? tIgHT~
// LoCk? TiGhT~ TIght~
// DIvULGe~ nObODY~ noBOdy?
// foreVER~ Tight~ GUaRd?
try
{
// Arrange.
Setup();
// Act.
_hc.SecretMasker.AddValue("TeLL? noboDy~ SEcreT?");
// The above string has the following Base64 variations based on the chop leading byte(s) method of Base64 aliasing:
var base64Variations = new[]
{
"VGVMTD8gbm9ib0R5fiBTRWNyZVQ/",
"ZUxMPyBub2JvRHl+IFNFY3JlVD8",
"TEw/IG5vYm9EeX4gU0VjcmVUPw",
// RFC 4648 (URL-safe Base64)
"VGVMTD8gbm9ib0R5fiBTRWNyZVQ_",
"ZUxMPyBub2JvRHl-IFNFY3JlVD8",
"TEw_IG5vYm9EeX4gU0VjcmVUPw"
};
var bookends = new[]
{
(string.Empty, string.Empty),
(string.Empty, "="),
(string.Empty, "=="),
(string.Empty, "==="),
("a", "z"),
("A", "Z"),
("abc", "abc"),
("ABC", "ABC"),
("0", "0"),
("00", "00"),
("000", "000"),
("123", "789"),
("`", "`"),
("'", "'"),
("\"", "\""),
("[", "]"),
("(", ")"),
("$(", ")"),
("{", "}"),
("${", "}"),
("!", "!"),
("!!", "!!"),
("%", "%"),
("%%", "%%"),
("_", "_"),
("__", "__"),
(":", ":"),
("::", "::"),
(";", ";"),
(";;", ";;"),
(":", string.Empty),
(";", string.Empty),
(string.Empty, ":"),
(string.Empty, ";"),
("VGVMTD8gbm9ib", "ZUxMPy"),
("VGVMTD8gbm9ib", "TEw/IG5vYm9EeX4"),
("ZUxMPy", "TEw/IG5vYm9EeX4"),
("VGVMTD8gbm9ib", string.Empty),
("TEw/IG5vYm9EeX4", string.Empty),
("ZUxMPy", string.Empty),
(string.Empty, "VGVMTD8gbm9ib"),
(string.Empty, "TEw/IG5vYm9EeX4"),
(string.Empty, "ZUxMPy"),
};
foreach (var variation in base64Variations)
{
foreach (var pair in bookends)
{
var (prefix, suffix) = pair;
var expected = string.Format("{0}{1}{2}", prefix, EXPECTED_SECRET_MASK, suffix);
var payload = string.Format("{0}{1}{2}", prefix, variation, suffix);
Assert.Equal(expected, _hc.SecretMasker.MaskSecrets(payload));
}
// Verify no masking is performed on a partial match.
for (int i = 1; i < variation.Length - 1; i++)
{
var fragment = variation[..i];
Assert.Equal(fragment, _hc.SecretMasker.MaskSecrets(fragment));
}
}
}
finally
{
// Cleanup.
Teardown();
}
}
[Theory] [Theory]
[InlineData("secret&secret&secret", "secret&secret&\x0033[96msecret\x0033[0m", "***\x0033[96m***\x0033[0m")] [InlineData("secret&secret&secret", "secret&secret&\x0033[96msecret\x0033[0m", "***\x0033[96m***\x0033[0m")]
[InlineData("secret&secret+secret", "secret&\x0033[96msecret+secret\x0033[0m", "***\x0033[96m***\x0033[0m")] [InlineData("secret&secret+secret", "secret&\x0033[96msecret+secret\x0033[0m", "***\x0033[96m***\x0033[0m")]

View File

@@ -1,4 +1,4 @@
using GitHub.Runner.Common.Util; using GitHub.Runner.Common.Util;
using System; using System;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Globalization; using System.Globalization;
@@ -56,9 +56,12 @@ namespace GitHub.Runner.Common.Tests
} }
var traceListener = new HostTraceListener(TraceFileName); var traceListener = new HostTraceListener(TraceFileName);
_secretMasker = new SecretMasker(); var encoders = new List<ValueEncoder>()
_secretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape); {
_secretMasker.AddValueEncoder(ValueEncoders.UriDataEscape); ValueEncoders.JsonStringEscape,
ValueEncoders.UriDataEscape
};
_secretMasker = new SecretMasker(encoders);
_traceManager = new TraceManager(traceListener, null, _secretMasker); _traceManager = new TraceManager(traceListener, null, _secretMasker);
_trace = GetTrace(nameof(TestHostContext)); _trace = GetTrace(nameof(TestHostContext));