Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Code exploration. #5056

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Code exploration.
michaelcfanning committed Dec 3, 2024
commit 160f61cbbca14f8ce7e797d3cdbd5a9c3173b0bb
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
@@ -22,13 +22,13 @@ Written for .NET Core in C#.
|![Win-x86](docs/res/win_med.png) **Windows x86**|[![Build & Test][win-x86-build-badge]][build]|
|![Win-arm64](docs/res/win_med.png) **Windows ARM64**|[![Build & Test][win-arm64-build-badge]][build]|
|![macOS](docs/res/apple_med.png) **macOS**|[![Build & Test][macOS-build-badge]][build]|
|![Linux-x64](docs/res/linux_med.png) **Linux x64**|[![Build & Test][linux-x64-build-badge]][build]|
|![Linux-arm](docs/res/linux_med.png) **Linux ARM**|[![Build & Test][linux-arm-build-badge]][build]|
|![RHEL6-x64](docs/res/redhat_med.png) **RHEL 6 x64**|[![Build & Test][rhel6-x64-build-badge]][build]|
|![Linux-x64](docs/res/linux_med.png) **Linux x64**|[![Build & Test][linux-x64-build-badge]][build]|
|![Linux-arm](docs/res/linux_med.png) **Linux ARM**|[![Build & Test][linux-arm-build-badge]][build]|
|![RHEL6-x64](docs/res/redhat_med.png) **RHEL 6 x64**|[![Build & Test][rhel6-x64-build-badge]][build]|

[win-x64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(x64)
[win-x86-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(x86)
[win-arm64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(ARM64)
[win-arm64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(arm64)
[macOS-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=macOS%20(x64)
[linux-x64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Linux%20(x64)
[linux-arm-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Linux%20(ARM)
10 changes: 9 additions & 1 deletion src/Agent.Sdk/Agent.Sdk.csproj
Original file line number Diff line number Diff line change
@@ -23,4 +23,12 @@
<PackageReference Include="System.Security.Cryptography.Xml" Version="6.0.1" />
<PackageReference Include="Azure.Core" Version="1.44.1" />
</ItemGroup>
</Project>

<ItemGroup>
<Compile Include="D:\src\su0\src\Microsoft.Security.Utilities.Core\*.cs" />
<Compile Include="D:\src\su0\src\Microsoft.Security.Utilities.Core\PotentialSecurityKeys\*.cs" />
<Compile Include="D:\src\su0\src\Microsoft.Security.Utilities.Core\PreciselyClassifiedSecurityKeys\*.cs" />
</ItemGroup>


</Project>
11 changes: 7 additions & 4 deletions src/Agent.Sdk/SecretMasking/ILoggedSecretMasker.cs
Original file line number Diff line number Diff line change
@@ -1,21 +1,24 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
//using Microsoft.TeamFoundation.DistributedTask.Logging;
using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
using System;
using Microsoft.Security.Utilities;

using ISecretMaskerTfs = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;

namespace Agent.Sdk.SecretMasking
{
/// <summary>
/// Extended ISecretMasker interface that is adding support of logging secret masker methods
/// </summary>
public interface ILoggedSecretMasker : ISecretMasker
public interface ILoggedSecretMasker : ISecretMasker, ISecretMaskerTfs
{
static int MinSecretLengthLimit { get; }
static int MinimumSecretLength { get; }

void AddRegex(String pattern, string origin);
void AddValue(String value, string origin);
void AddValueEncoder(ValueEncoder encoder, string origin);
void AddValueEncoder(LiteralEncoder encoder, string origin);
void SetTrace(ITraceWriter trace);
new string MaskSecrets(string input);
}
}
60 changes: 60 additions & 0 deletions src/Agent.Sdk/SecretMasking/LiteralEncoders.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
//using Microsoft.TeamFoundation.DistributedTask.Logging;
using Newtonsoft.Json;

using System;
using System.Text;

namespace Agent.Sdk.SecretMasking
{
public static class LiteralEncoders
{
public static String JsonStringEscape(String value)
{
// Convert to a JSON string and then remove the leading/trailing double-quote.
String jsonString = JsonConvert.ToString(value);
String jsonEscapedValue = jsonString.Substring(startIndex: 1, length: jsonString.Length - 2);
return jsonEscapedValue;
}

public static String BackslashEscape(String value)
{
return value.Replace(@"\\", @"\").Replace(@"\'", @"'").Replace(@"\""", @"""").Replace(@"\t", "\t");
}

public static String UriDataEscape(String value)
{
return UriDataEscape(value, 65519);
}

internal static String UriDataEscape(
String value,
Int32 maxSegmentSize)
{
if (value.Length <= maxSegmentSize)
{
return Uri.EscapeDataString(value);
}

// Workaround size limitation in Uri.EscapeDataString
var result = new StringBuilder();
var i = 0;
do
{
var length = Math.Min(value.Length - i, maxSegmentSize);

if (Char.IsHighSurrogate(value[i + length - 1]) && length > 1)
{
length--;
}

result.Append(Uri.EscapeDataString(value.Substring(i, length)));
i += length;
}
while (i < value.Length);

return result.ToString();
}
}
}
57 changes: 43 additions & 14 deletions src/Agent.Sdk/SecretMasking/LoggedSecretMasker.cs
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
using System;
using ValueEncoder = Microsoft.TeamFoundation.DistributedTask.Logging.ValueEncoder;
using ISecretMaskerVSO = Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker;
using Microsoft.Security.Utilities;
using System.Collections.Generic;
using System;

namespace Agent.Sdk.SecretMasking
{
@@ -12,7 +13,7 @@ namespace Agent.Sdk.SecretMasking
/// </summary>
public class LoggedSecretMasker : ILoggedSecretMasker
{
private ISecretMasker _secretMasker;
private SecretMasker _secretMasker;
private ITraceWriter _trace;


@@ -21,7 +22,7 @@ private void Trace(string msg)
this._trace?.Info(msg);
}

public LoggedSecretMasker(ISecretMasker secretMasker)
public LoggedSecretMasker(SecretMasker secretMasker)
{
this._secretMasker = secretMasker;
}
@@ -54,7 +55,6 @@ public void AddValue(string value, string origin)
}
public void AddRegex(string pattern)
{
this._secretMasker.AddRegex(pattern);
}

/// <summary>
@@ -82,30 +82,30 @@ public int MinSecretLength
{
get
{
return _secretMasker.MinSecretLength;
return _secretMasker.MinimumSecretLength;
}
set
{
if (value > MinSecretLengthLimit)
{
_secretMasker.MinSecretLength = MinSecretLengthLimit;
_secretMasker.MinimumSecretLength = MinSecretLengthLimit;
}
else
{
_secretMasker.MinSecretLength = value;
_secretMasker.MinimumSecretLength = value;
}
}
}

public void RemoveShortSecretsFromDictionary()
{
this._trace?.Info("Removing short secrets from masking dictionary");
_secretMasker.RemoveShortSecretsFromDictionary();
_secretMasker.RemovePatternsThatDoNotMeetLengthLimits();
}

public void AddValueEncoder(ValueEncoder encoder)
public void AddValueEncoder(LiteralEncoder encoder)
{
this._secretMasker.AddValueEncoder(encoder);
this._secretMasker.AddLiteralEncoder(encoder);
}


@@ -114,7 +114,7 @@ public void AddValueEncoder(ValueEncoder encoder)
/// </summary>
/// <param name="encoder"></param>
/// <param name="origin"></param>
public void AddValueEncoder(ValueEncoder encoder, string origin)
public void AddValueEncoder(LiteralEncoder encoder, string origin)
{
this.Trace($"Setting up value for origin: {origin}");
this.Trace($"Length: {encoder.ToString().Length}.");
@@ -129,14 +129,43 @@ public void AddValueEncoder(ValueEncoder encoder, string origin)

public ISecretMasker Clone()
{
return new LoggedSecretMasker(this._secretMasker.Clone());
return null;
//return new LoggedSecretMasker(this._secretMasker.Clone());
}

public string MaskSecrets(string input)
{
return this._secretMasker.MaskSecrets(input);
}

ISecretMaskerVSO ISecretMaskerVSO.Clone() => this.Clone();
public IEnumerable<Detection> DetectSecrets(string input)
{
return this._secretMasker.DetectSecrets(input);
}

public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
this._secretMasker?.Dispose();
this._secretMasker = null;
}
}

public void AddValueEncoder(ValueEncoder encoder)
{
//return this.AddValueEncoder((LiteralEncoder)encoder)
}

Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker Microsoft.TeamFoundation.DistributedTask.Logging.ISecretMasker.Clone()
{
// WRONG: should do this return new LoggedSecretMasker(this._secretMasker.Clone());
return new LoggedSecretMasker(this._secretMasker);
}
}
}
2 changes: 1 addition & 1 deletion src/Agent.Worker/ContainerOperationProvider.cs
Original file line number Diff line number Diff line change
@@ -298,7 +298,7 @@ private async Task<string> GetAcrPasswordFromAADToken(IExecutionContext executio
}

// Mark retrieved password as secret
HostContext.SecretMasker.AddValue(AcrPassword);
HostContext.SecretMasker.AddValue(AcrPassword, origin: null);

return AcrPassword;
}
8 changes: 0 additions & 8 deletions src/Agent.Worker/ExecutionContext.cs
Original file line number Diff line number Diff line change
@@ -567,14 +567,6 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation
PrependPath = new List<string>();

var minSecretLen = AgentKnobs.MaskedSecretMinLength.GetValue(this).AsInt();
HostContext.SecretMasker.MinSecretLength = minSecretLen;

if (HostContext.SecretMasker.MinSecretLength < minSecretLen)
{
warnings.Add(StringUtil.Loc("MinSecretsLengtLimitWarning", HostContext.SecretMasker.MinSecretLength));
}

HostContext.SecretMasker.RemoveShortSecretsFromDictionary();

// Docker (JobContainer)
string imageName = Variables.Get("_PREVIEW_VSTS_DOCKER_IMAGE");
Original file line number Diff line number Diff line change
@@ -36,7 +36,6 @@ public class LegacyTestRunDataPublisher : AgentService, ILegacyTestRunDataPublis
private int _runCounter = 0;
private IFeatureFlagService _featureFlagService;
private bool _calculateTestRunSummary;
private bool _isFlakyCheckEnabled;
private string _testRunner;
private ITestResultsServer _testResultsServer;
private TestRunDataPublisherHelper _testRunPublisherHelper;
@@ -55,7 +54,6 @@ public void InitializePublisher(IExecutionContext context, string projectName, V
_testResultsServer = HostContext.GetService<ITestResultsServer>();
_testResultsServer.InitializeServer(connection, _executionContext);
_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
_isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);
_testRunPublisherHelper = new TestRunDataPublisherHelper(_executionContext, null, _testRunPublisher, _testResultsServer);
Trace.Leaving();
}
@@ -210,7 +208,9 @@ private async Task<bool> PublishAllTestResultsToSingleTestRunAsync(List<string>

// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);

if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
IList<TestRun> publishedRuns = new List<TestRun>();
publishedRuns.Add(updatedRun);
@@ -313,7 +313,9 @@ private async Task<bool> PublishToNewTestRunPerTestResultFileAsync(List<string>

// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);

if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
15 changes: 10 additions & 5 deletions src/Agent.Worker/TestResults/TestDataPublisher.cs
Original file line number Diff line number Diff line change
@@ -41,7 +41,6 @@ public sealed class TestDataPublisher : AgentService, ITestDataPublisher
private IFeatureFlagService _featureFlagService;
private string _testRunner;
private bool _calculateTestRunSummary;
private bool _isFlakyCheckEnabled;
private TestRunDataPublisherHelper _testRunPublisherHelper;
private ITestResultsServer _testResultsServer;

@@ -59,8 +58,6 @@ public void InitializePublisher(IExecutionContext context, string projectName, V
var extensionManager = HostContext.GetService<IExtensionManager>();
_featureFlagService = HostContext.GetService<IFeatureFlagService>();
_featureFlagService.InitializeFeatureService(_executionContext, connection);
_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);
_isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid); ;
_parser = (extensionManager.GetExtensions<IParser>()).FirstOrDefault(x => _testRunner.Equals(x.Name, StringComparison.OrdinalIgnoreCase));
_testRunPublisherHelper = new TestRunDataPublisherHelper(_executionContext, _testRunPublisher, null, _testResultsServer);
Trace.Leaving();
@@ -89,6 +86,8 @@ public void InitializePublisher(IExecutionContext context, string projectName, V

IList<TestRun> publishedRuns = publishtestRunDataTask.Result;

_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);

var isTestRunOutcomeFailed = GetTestRunOutcome(_executionContext, testRunData, out TestRunSummary testRunSummary);

// Storing testrun summary in environment variable, which will be read by PublishPipelineMetadataTask and publish to evidence store.
@@ -99,7 +98,9 @@ public void InitializePublisher(IExecutionContext context, string projectName, V

// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);

if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
@@ -187,6 +188,8 @@ public void InitializePublisher(IExecutionContext context, string projectName, V

IList<TestRun> publishedRuns = publishtestRunDataTask.Result;

_calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid);

var isTestRunOutcomeFailed = GetTestRunOutcome(_executionContext, testRunData, out TestRunSummary testRunSummary);

// Storing testrun summary in environment variable, which will be read by PublishPipelineMetadataTask and publish to evidence store.
@@ -197,7 +200,9 @@ public void InitializePublisher(IExecutionContext context, string projectName, V

// Check failed results for flaky aware
// Fallback to flaky aware if there are any failures.
if (isTestRunOutcomeFailed && _isFlakyCheckEnabled)
bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid);

if (isTestRunOutcomeFailed && isFlakyCheckEnabled)
{
var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName);
if (runOutcome != null && runOutcome.HasValue)
Loading