diff --git a/.CodeQL.yml b/.CodeQL.yml new file mode 100644 index 0000000000..c06c8e53c3 --- /dev/null +++ b/.CodeQL.yml @@ -0,0 +1,16 @@ +path_classifiers: + test: + # Note: use only forward slash / as a path separator. + # * Matches any sequence of characters except a forward slash. + # ** Matches any sequence of characters, including a forward slash. + # This wildcard must either be surrounded by forward slash symbols, or used as the first segment of a path. + # It matches zero or more whole directory segments. There is no need to use a wildcard at the end of a directory path because all sub-directories are automatically matched. + # That is, /anything/ matches the anything directory and all its subdirectories. + # Always enclose the expression in double quotes if it includes *. + - src/Test + + # The default behavior is to tag all files created during the + # build as `generated`. Results are hidden for generated code. You can tag + # further files as being generated by adding them to the `generated` section. + generated: + - _reports diff --git a/.azure-pipelines/build-job.yml b/.azure-pipelines/build-job.yml index 02c5233de2..76d4b4977f 100644 --- a/.azure-pipelines/build-job.yml +++ b/.azure-pipelines/build-job.yml @@ -149,7 +149,7 @@ jobs: ) # 1ES images used on the ARM pool doesn't contain unzip tool, so we need to install it before starting the build - - ${{ if and(eq(parameters.arch, 'arm64'), ne(parameters.os, 'osx')) }}: + - ${{ if and(eq(parameters.arch, 'arm64'), ne(parameters.os, 'osx'), ne(parameters.os, 'win')) }}: - script: sudo dnf -y update && sudo dnf -y install unzip displayName: Install unzip retryCountOnTaskFailure: 5 diff --git a/.azure-pipelines/pipeline.yml b/.azure-pipelines/pipeline.yml index cdb2d004ce..c1c5dd6a77 100644 --- a/.azure-pipelines/pipeline.yml +++ b/.azure-pipelines/pipeline.yml @@ -42,6 +42,9 @@ parameters: - name: win_x86 type: boolean default: true +- name: win_arm64 + type: boolean + default: false - name: linux_x64 type: boolean default: true @@ -182,6 +185,26 @@ extends: publishArtifacts: ${{ parameters.publishArtifacts }} buildAlternatePackage: ${{ parameters.buildAlternatePackage }} targetFramework: ${{ parameters.targetFramework }} + + # Windows (ARM64) + - ${{ if parameters.win_arm64 }}: + - template: /.azure-pipelines/build-jobs.yml@self + parameters: + jobName: build_windows_arm64 + displayName: Windows (ARM64) + pool: + name: 1ES-ABTT-Shared-Pool + image: abtt-windows-2022 + os: windows + os: win + arch: arm64 + branch: ${{ parameters.branch }} + unitTests: false + functionalTests: false + sign: ${{ parameters.sign }} + publishArtifacts: ${{ parameters.publishArtifacts }} + buildAlternatePackage: ${{ parameters.buildAlternatePackage }} + targetFramework: ${{ parameters.targetFramework }} # Linux (x64) - ${{ if parameters.linux_x64 }}: diff --git a/.editorconfig b/.editorconfig index 85323ba60a..e0c9ec240f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -59,7 +59,7 @@ dotnet_diagnostic.IDE0007.severity = none dotnet_diagnostic.IDE0008.severity = none # IDE0010: Add missing cases -dotnet_diagnostic.IDE0010.severity = warning +dotnet_diagnostic.IDE0010.severity = none # IDE0011: Add braces dotnet_diagnostic.IDE0011.severity = suggestion diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 90f5aacd68..219b855062 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ # Global rule: -* @microsoft/akvelon-build-task-team @microsoft/azure-pipelines-platform +* @microsoft/azure-pipelines-tasks-and-agent @microsoft/azure-pipelines-platform diff --git a/.vsts.ci.yml b/.vsts.ci.yml index c3f4bfe6a2..4acd3308bb 100644 --- a/.vsts.ci.yml +++ b/.vsts.ci.yml @@ -14,6 +14,10 @@ parameters: type: boolean displayName: Windows (x86) default: true +- name: win_arm64 + type: boolean + displayName: Windows (ARM64) + default: false - name: linux_x64 type: boolean displayName: Linux (x64) @@ -59,6 +63,7 @@ extends: testProxyAgent: ${{ parameters.testProxyAgent }} win_x64: ${{ parameters.win_x64 }} win_x86: ${{ parameters.win_x86 }} + win_arm64: ${{ parameters.win_arm64 }} linux_x64: ${{ parameters.linux_x64 }} linux_arm: ${{ parameters.linux_arm }} linux_arm64: ${{ parameters.linux_arm64 }} diff --git a/README.md b/README.md index be37a52a33..d06da54415 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,15 @@ Written for .NET Core in C#. |---|:-----:| |![Win-x64](docs/res/win_med.png) **Windows x64**|[![Build & Test][win-x64-build-badge]][build]| |![Win-x86](docs/res/win_med.png) **Windows x86**|[![Build & Test][win-x86-build-badge]][build]| +|![Win-arm64](docs/res/win_med.png) **Windows ARM64**|[![Build & Test][win-arm64-build-badge]][build]| |![macOS](docs/res/apple_med.png) **macOS**|[![Build & Test][macOS-build-badge]][build]| -|![Linux-x64](docs/res/linux_med.png) **Linux x64**|[![Build & Test][linux-x64-build-badge]][build]| -|![Linux-arm](docs/res/linux_med.png) **Linux ARM**|[![Build & Test][linux-arm-build-badge]][build]| -|![RHEL6-x64](docs/res/redhat_med.png) **RHEL 6 x64**|[![Build & Test][rhel6-x64-build-badge]][build]| +|![Linux-x64](docs/res/linux_med.png) **Linux x64**|[![Build & Test][linux-x64-build-badge]][build]| +|![Linux-arm](docs/res/linux_med.png) **Linux ARM**|[![Build & Test][linux-arm-build-badge]][build]| +|![RHEL6-x64](docs/res/redhat_med.png) **RHEL 6 x64**|[![Build & Test][rhel6-x64-build-badge]][build]| [win-x64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(x64) [win-x86-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(x86) +[win-arm64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Windows%20(ARM64) [macOS-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=macOS%20(x64) [linux-x64-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Linux%20(x64) [linux-arm-build-badge]: https://mseng.visualstudio.com/pipelinetools/_apis/build/status/VSTS.Agent/azure-pipelines-agent.ci?branchName=master&jobname=Linux%20(ARM) diff --git a/assets.json b/assets.json index de0750d57f..c8c2f1f21e 100644 --- a/assets.json +++ b/assets.json @@ -23,6 +23,18 @@ "version": "", "downloadUrl": "https://vstsagentpackage.azureedge.net/agent//pipelines-agent-win-x86-.zip" }, + { + "name": "vsts-agent-win-arm64-.zip", + "platform": "win-arm64", + "version": "", + "downloadUrl": "https://vstsagentpackage.azureedge.net/agent//vsts-agent-win-arm64-.zip" + }, + { + "name": "pipelines-agent-win-arm64-.zip", + "platform": "win-arm64", + "version": "", + "downloadUrl": "https://vstsagentpackage.azureedge.net/agent//pipelines-agent-win-arm64-.zip" + }, { "name": "vsts-agent-osx-x64-.tar.gz", "platform": "osx-x64", diff --git a/src/Agent.Listener/net6.json b/src/Agent.Listener/net6.json index bcb3262b23..a6964b28de 100644 --- a/src/Agent.Listener/net6.json +++ b/src/Agent.Listener/net6.json @@ -71,6 +71,22 @@ } ] }, + { + "id": "ol", + "versions": [ + { + "name": "7+" + } + ] + }, + { + "id": "rocky", + "versions": [ + { + "name": "8+" + } + ] + }, { "id": "sles", "versions": [ diff --git a/src/Agent.Listener/net8.json b/src/Agent.Listener/net8.json index bdc9c1aec3..2d2600cd66 100644 --- a/src/Agent.Listener/net8.json +++ b/src/Agent.Listener/net8.json @@ -34,14 +34,6 @@ } ] }, - { - "id": "ol", - "versions": [ - { - "name": "8+" - } - ] - }, { "id": "macOS", "versions": [ @@ -82,6 +74,22 @@ } ] }, + { + "id": "ol", + "versions": [ + { + "name": "8+" + } + ] + }, + { + "id": "rocky", + "versions": [ + { + "name": "8+" + } + ] + }, { "id": "sles", "versions": [ diff --git a/src/Agent.Worker/TestResults/Legacy/LegacyTestRunDataPublisher.cs b/src/Agent.Worker/TestResults/Legacy/LegacyTestRunDataPublisher.cs index 63f17d87f0..a71775305e 100644 --- a/src/Agent.Worker/TestResults/Legacy/LegacyTestRunDataPublisher.cs +++ b/src/Agent.Worker/TestResults/Legacy/LegacyTestRunDataPublisher.cs @@ -36,6 +36,7 @@ public class LegacyTestRunDataPublisher : AgentService, ILegacyTestRunDataPublis private int _runCounter = 0; private IFeatureFlagService _featureFlagService; private bool _calculateTestRunSummary; + private bool _isFlakyCheckEnabled; private string _testRunner; private ITestResultsServer _testResultsServer; private TestRunDataPublisherHelper _testRunPublisherHelper; @@ -54,6 +55,7 @@ public void InitializePublisher(IExecutionContext context, string projectName, V _testResultsServer = HostContext.GetService(); _testResultsServer.InitializeServer(connection, _executionContext); _calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid); + _isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid); _testRunPublisherHelper = new TestRunDataPublisherHelper(_executionContext, null, _testRunPublisher, _testResultsServer); Trace.Leaving(); } @@ -208,9 +210,7 @@ private async Task PublishAllTestResultsToSingleTestRunAsync(List // Check failed results for flaky aware // Fallback to flaky aware if there are any failures. - bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid); - - if (isTestRunOutcomeFailed && isFlakyCheckEnabled) + if (isTestRunOutcomeFailed && _isFlakyCheckEnabled) { IList publishedRuns = new List(); publishedRuns.Add(updatedRun); @@ -313,9 +313,7 @@ private async Task PublishToNewTestRunPerTestResultFileAsync(List // Check failed results for flaky aware // Fallback to flaky aware if there are any failures. - bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid); - - if (isTestRunOutcomeFailed && isFlakyCheckEnabled) + if (isTestRunOutcomeFailed && _isFlakyCheckEnabled) { var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName); if (runOutcome != null && runOutcome.HasValue) diff --git a/src/Agent.Worker/TestResults/TestDataPublisher.cs b/src/Agent.Worker/TestResults/TestDataPublisher.cs index af99350378..5d483e3837 100644 --- a/src/Agent.Worker/TestResults/TestDataPublisher.cs +++ b/src/Agent.Worker/TestResults/TestDataPublisher.cs @@ -41,6 +41,7 @@ public sealed class TestDataPublisher : AgentService, ITestDataPublisher private IFeatureFlagService _featureFlagService; private string _testRunner; private bool _calculateTestRunSummary; + private bool _isFlakyCheckEnabled; private TestRunDataPublisherHelper _testRunPublisherHelper; private ITestResultsServer _testResultsServer; @@ -58,6 +59,8 @@ public void InitializePublisher(IExecutionContext context, string projectName, V var extensionManager = HostContext.GetService(); _featureFlagService = HostContext.GetService(); _featureFlagService.InitializeFeatureService(_executionContext, connection); + _calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid); + _isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid); ; _parser = (extensionManager.GetExtensions()).FirstOrDefault(x => _testRunner.Equals(x.Name, StringComparison.OrdinalIgnoreCase)); _testRunPublisherHelper = new TestRunDataPublisherHelper(_executionContext, _testRunPublisher, null, _testResultsServer); Trace.Leaving(); @@ -86,8 +89,6 @@ public void InitializePublisher(IExecutionContext context, string projectName, V IList publishedRuns = publishtestRunDataTask.Result; - _calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid); - var isTestRunOutcomeFailed = GetTestRunOutcome(_executionContext, testRunData, out TestRunSummary testRunSummary); // Storing testrun summary in environment variable, which will be read by PublishPipelineMetadataTask and publish to evidence store. @@ -98,9 +99,7 @@ public void InitializePublisher(IExecutionContext context, string projectName, V // Check failed results for flaky aware // Fallback to flaky aware if there are any failures. - bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid); - - if (isTestRunOutcomeFailed && isFlakyCheckEnabled) + if (isTestRunOutcomeFailed && _isFlakyCheckEnabled) { var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName); if (runOutcome != null && runOutcome.HasValue) @@ -154,6 +153,7 @@ public void InitializePublisher(IExecutionContext context, string projectName, V for (testRunDataIterator = 0; testRunDataIterator < testRunData.Count; testRunDataIterator++) { + var testResultsUpdated = new List(); for (testResultDataIterator = 0; testResultDataIterator < testRunData[testRunDataIterator].TestResults.Count; testResultDataIterator++) { var testResultFQN = testRunData[testRunDataIterator].TestResults[testResultDataIterator].AutomatedTestStorage + @@ -161,17 +161,23 @@ public void InitializePublisher(IExecutionContext context, string projectName, V if (testResultByFQN.TryGetValue(testResultFQN, out List inputs)) { - testRunData[testRunDataIterator].TestResults[testResultDataIterator].TestPoint = inputs[0].TestPoint; - testRunData[testRunDataIterator].TestResults[testResultDataIterator].TestCaseTitle = inputs[0].TestCaseTitle; - testRunData[testRunDataIterator].TestResults[testResultDataIterator].Configuration = inputs[0].Configuration; - testRunData[testRunDataIterator].TestResults[testResultDataIterator].TestCase = inputs[0].TestCase; - testRunData[testRunDataIterator].TestResults[testResultDataIterator].Owner = inputs[0].Owner; - testRunData[testRunDataIterator].TestResults[testResultDataIterator].State = "5"; - testRunData[testRunDataIterator].TestResults[testResultDataIterator].TestCaseRevision = inputs[0].TestCaseRevision; - - testResultByFQN[testResultFQN].RemoveAt(0); + foreach (var input in inputs) + { + var testCaseResultDataUpdated = TestResultUtils.CloneTestCaseResultData(testRunData[testRunDataIterator].TestResults[testResultDataIterator]); + + testCaseResultDataUpdated.TestPoint = input.TestPoint; + testCaseResultDataUpdated.TestCaseTitle = input.TestCaseTitle; + testCaseResultDataUpdated.Configuration = input.Configuration; + testCaseResultDataUpdated.TestCase = input.TestCase; + testCaseResultDataUpdated.Owner = input.Owner; + testCaseResultDataUpdated.State = "5"; + testCaseResultDataUpdated.TestCaseRevision = input.TestCaseRevision; + + testResultsUpdated.Add(testCaseResultDataUpdated); + } } } + testRunData[testRunDataIterator].TestResults = testResultsUpdated; } } @@ -188,8 +194,6 @@ public void InitializePublisher(IExecutionContext context, string projectName, V IList publishedRuns = publishtestRunDataTask.Result; - _calculateTestRunSummary = _featureFlagService.GetFeatureFlagState(TestResultsConstants.CalculateTestRunSummaryFeatureFlag, TestResultsConstants.TFSServiceInstanceGuid); - var isTestRunOutcomeFailed = GetTestRunOutcome(_executionContext, testRunData, out TestRunSummary testRunSummary); // Storing testrun summary in environment variable, which will be read by PublishPipelineMetadataTask and publish to evidence store. @@ -200,9 +204,7 @@ public void InitializePublisher(IExecutionContext context, string projectName, V // Check failed results for flaky aware // Fallback to flaky aware if there are any failures. - bool isFlakyCheckEnabled = _featureFlagService.GetFeatureFlagState(TestResultsConstants.EnableFlakyCheckInAgentFeatureFlag, TestResultsConstants.TCMServiceInstanceGuid); - - if (isTestRunOutcomeFailed && isFlakyCheckEnabled) + if (isTestRunOutcomeFailed && _isFlakyCheckEnabled) { var runOutcome = _testRunPublisherHelper.CheckRunsForFlaky(publishedRuns, _projectName); if (runOutcome != null && runOutcome.HasValue) diff --git a/src/Agent.Worker/TestResults/Utils/TestResultUtils.cs b/src/Agent.Worker/TestResults/Utils/TestResultUtils.cs index 5794409a42..cc49b015df 100644 --- a/src/Agent.Worker/TestResults/Utils/TestResultUtils.cs +++ b/src/Agent.Worker/TestResults/Utils/TestResultUtils.cs @@ -5,6 +5,8 @@ using System; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; +using Microsoft.TeamFoundation.TestClient.PublishTestResults; +using System.Linq; namespace Microsoft.VisualStudio.Services.Agent.Worker.TestResults.Utils { @@ -32,6 +34,61 @@ public static void StoreTestRunSummaryInEnvVar(IExecutionContext executionContex } } + public static TestCaseResultData CloneTestCaseResultData(TestCaseResultData original) + { + return new TestCaseResultData + { + Id = original.Id, + Comment = original.Comment, + Configuration = original.Configuration, + Project = original.Project, + StartedDate = original.StartedDate, + CompletedDate = original.CompletedDate, + DurationInMs = original.DurationInMs, + Outcome = original.Outcome, + Revision = original.Revision, + State = original.State, + TestCase = original.TestCase, + TestPoint = original.TestPoint, + TestRun = original.TestRun, + ResolutionStateId = original.ResolutionStateId, + ResolutionState = original.ResolutionState, + LastUpdatedDate = original.LastUpdatedDate, + Priority = original.Priority, + ComputerName = original.ComputerName, + ResetCount = original.ResetCount, + Build = original.Build, + Release = original.Release, + ErrorMessage = original.ErrorMessage, + CreatedDate = original.CreatedDate, + IterationDetails = original.IterationDetails?.ToList(), + AssociatedBugs = original.AssociatedBugs?.ToList(), + Url = original.Url, + FailureType = original.FailureType, + AutomatedTestName = original.AutomatedTestName, + AutomatedTestStorage = original.AutomatedTestStorage, + AutomatedTestType = original.AutomatedTestType, + AutomatedTestTypeId = original.AutomatedTestTypeId, + AutomatedTestId = original.AutomatedTestId, + Area = original.Area, + TestCaseTitle = original.TestCaseTitle, + StackTrace = original.StackTrace, + CustomFields = original.CustomFields?.ToList(), + BuildReference = original.BuildReference, + ReleaseReference = original.ReleaseReference, + TestPlan = original.TestPlan, + TestSuite = original.TestSuite, + TestCaseReferenceId = original.TestCaseReferenceId, + Owner = original.Owner, + RunBy = original.RunBy, + LastUpdatedBy = original.LastUpdatedBy, + ResultGroupType = original.ResultGroupType, + TestCaseRevision = original.TestCaseRevision, + TestCaseSubResultData = original.TestCaseSubResultData?.ToList(), + AttachmentData = original.AttachmentData + }; + } + private static string GetEvidenceStoreMetadata(IExecutionContext executionContext, TestRunSummary testRunSummary, string testRunner, string name, string description) { string evidenceStoreMetadataString = string.Empty; diff --git a/src/Agent.Worker/Worker.cs b/src/Agent.Worker/Worker.cs index cef1e00b46..a0fed0475f 100644 --- a/src/Agent.Worker/Worker.cs +++ b/src/Agent.Worker/Worker.cs @@ -174,6 +174,20 @@ private void InitializeSecretMasker(Pipelines.AgentJobRequestMessage message) var escapedSecret2 = variable.Value.Value.Replace("\r", "%0D") .Replace("\n", "%0A"); AddUserSuppliedSecret(escapedSecret2); + // We need to mask the base 64 value of the secret as well + var base64Secret = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(variable.Value.Value)); + // Add the base64 secret to the secret masker + AddUserSuppliedSecret(base64Secret); + // also, we escape some characters for variables when we print them out in debug mode. We need to + // add the escaped version of these secrets as well + var escapedSecret3 = base64Secret.Replace("%", "%AZP25") + .Replace("\r", "%0D") + .Replace("\n", "%0A"); + AddUserSuppliedSecret(escapedSecret3); + // Since % escaping may be turned off, also mask a version escaped with just newlines + var escapedSecret4 = base64Secret.Replace("\r", "%0D") + .Replace("\n", "%0A"); + AddUserSuppliedSecret(escapedSecret4); } } diff --git a/src/Common.props b/src/Common.props index 6251bbb91d..2f037412e7 100644 --- a/src/Common.props +++ b/src/Common.props @@ -1,5 +1,6 @@ + 10.0 net6.0;net8.0 $(PackageRuntime) true @@ -37,6 +38,13 @@ X86 + + ARM64 + + + ARM64 + + X64 @@ -64,7 +72,8 @@ win-x64 win-x86 - + win-arm64 + $(OSPlatform);$(OSArchitecture);$(DebugConstant);TRACE diff --git a/src/Misc/InstallAgentPackage.template.xml b/src/Misc/InstallAgentPackage.template.xml index 4c60a4f9bf..1581c8bae8 100644 --- a/src/Misc/InstallAgentPackage.template.xml +++ b/src/Misc/InstallAgentPackage.template.xml @@ -73,6 +73,16 @@ + + + + + + + + + + diff --git a/src/Misc/Publish.template.ps1 b/src/Misc/Publish.template.ps1 index 5d71b86336..64caf44c99 100644 --- a/src/Misc/Publish.template.ps1 +++ b/src/Misc/Publish.template.ps1 @@ -8,6 +8,8 @@ if ($pwd -notlike '*tfsgheus20') { Add-DistributedTaskPackage -PackageType agent -Platform win-x86 -Version -DownloadUrl https://vstsagentpackage.azureedge.net/agent//vsts-agent-win-x86-.zip -HashValue -InfoUrl https://go.microsoft.com/fwlink/?LinkId=798199 -Filename vsts-agent-win-x86-.zip + Add-DistributedTaskPackage -PackageType agent -Platform win-arm64 -Version -DownloadUrl https://vstsagentpackage.azureedge.net/agent//vsts-agent-win-arm64-.zip -HashValue -InfoUrl https://go.microsoft.com/fwlink/?LinkId=798199 -Filename vsts-agent-win-arm64-.zip + Add-DistributedTaskPackage -PackageType agent -Platform osx-x64 -Version -DownloadUrl https://vstsagentpackage.azureedge.net/agent//vsts-agent-osx-x64-.tar.gz -HashValue -InfoUrl https://go.microsoft.com/fwlink/?LinkId=798199 -Filename vsts-agent-osx-x64-.tar.gz Add-DistributedTaskPackage -PackageType agent -Platform linux-x64 -Version -DownloadUrl https://vstsagentpackage.azureedge.net/agent//vsts-agent-linux-x64-.tar.gz -HashValue -InfoUrl https://go.microsoft.com/fwlink/?LinkId=798199 -Filename vsts-agent-linux-x64-.tar.gz @@ -28,6 +30,8 @@ if ($pwd -notlike '*tfsgheus20') { Add-DistributedTaskPackage -PackageType pipelines-agent -Platform win-x86 -Version -DownloadUrl https://vstsagentpackage.azureedge.net/agent//pipelines-agent-win-x86-.zip -HashValue -InfoUrl https://go.microsoft.com/fwlink/?LinkId=798199 -Filename pipelines-agent-win-x86-.zip + Add-DistributedTaskPackage -PackageType pipelines-agent -Platform win-arm64 -Version -DownloadUrl https://vstsagentpackage.azureedge.net/agent//pipelines-agent-win-arm64-.zip -HashValue -InfoUrl https://go.microsoft.com/fwlink/?LinkId=798199 -Filename pipelines-agent-win-arm64-.zip + Add-DistributedTaskPackage -PackageType pipelines-agent -Platform osx-x64 -Version -DownloadUrl https://vstsagentpackage.azureedge.net/agent//pipelines-agent-osx-x64-.tar.gz -HashValue -InfoUrl https://go.microsoft.com/fwlink/?LinkId=798199 -Filename pipelines-agent-osx-x64-.tar.gz Add-DistributedTaskPackage -PackageType pipelines-agent -Platform linux-x64 -Version -DownloadUrl https://vstsagentpackage.azureedge.net/agent//pipelines-agent-linux-x64-.tar.gz -HashValue -InfoUrl https://go.microsoft.com/fwlink/?LinkId=798199 -Filename pipelines-agent-linux-x64-.tar.gz diff --git a/src/Misc/UpdateAgentPackage.template.xml b/src/Misc/UpdateAgentPackage.template.xml index 571c07e99b..27d7c87679 100644 --- a/src/Misc/UpdateAgentPackage.template.xml +++ b/src/Misc/UpdateAgentPackage.template.xml @@ -42,6 +42,11 @@ + + + + + @@ -77,5 +82,10 @@ + + + + + diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index 01c984ff77..0cc7f73bbc 100644 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -10,14 +10,22 @@ INCLUDE_NODE10=${INCLUDE_NODE10:-true} CONTAINER_URL=https://vstsagenttools.blob.core.windows.net/tools NODE_URL=https://nodejs.org/dist +NODE_UNOFFICIAL_URL=https://unofficial-builds.nodejs.org/download/release + if [[ "$PACKAGERUNTIME" == "linux-musl-x64" ]]; then NODE_URL=https://unofficial-builds.nodejs.org/download/release INCLUDE_NODE6=false fi +if [[ "$PACKAGERUNTIME" == "win-arm64" ]]; then + INCLUDE_NODE6=false + INCLUDE_NODE10=false; +fi + NODE_VERSION="6.17.1" NODE10_VERSION="10.24.1" NODE16_VERSION="16.20.2" +NODE16_WIN_ARM64_VERSION="16.9.1" NODE20_VERSION="20.17.0" MINGIT_VERSION="2.47.0.2" LFS_VERSION="3.4.0" @@ -155,13 +163,14 @@ function acquireExternalTool() { fi } +echo "PACKAGE RUNTIME: $PACKAGERUNTIME" + if [[ "$PACKAGERUNTIME" == "win-x"* ]]; then # Download external tools for Windows. BIT="32" if [[ "$PACKAGERUNTIME" == "win-x64" ]]; then BIT="64" - acquireExternalTool "$CONTAINER_URL/azcopy/1/azcopy.zip" azcopy acquireExternalTool "$CONTAINER_URL/vstshost/m122_887c6659_binding_redirect_patched/vstshost.zip" vstshost acquireExternalTool "$CONTAINER_URL/vstsom/m153_47c0856d_adhoc/vstsom.zip" vstsom @@ -187,6 +196,42 @@ if [[ "$PACKAGERUNTIME" == "win-x"* ]]; then acquireExternalTool "${NODE_URL}/v${NODE16_VERSION}/${PACKAGERUNTIME}/node.lib" node16/bin acquireExternalTool "${NODE_URL}/v${NODE20_VERSION}/${PACKAGERUNTIME}/node.exe" node20_1/bin acquireExternalTool "${NODE_URL}/v${NODE20_VERSION}/${PACKAGERUNTIME}/node.lib" node20_1/bin +elif [[ "$PACKAGERUNTIME" == "win-arm64" || "$PACKAGERUNTIME" == "win-arm32" ]]; then + # Download external tools for Windows ARM + + BIT="32" + if [[ "$PACKAGERUNTIME" == "win-arm64" ]]; then + BIT="64" + + # acquireExternalTool "$CONTAINER_URL/azcopy/1/azcopy.zip" azcopy # Unavailable for Win ARM 64 - https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10?tabs=dnf#download-the-azcopy-portable-binary + acquireExternalTool "$CONTAINER_URL/vstshost/m122_887c6659_binding_redirect_patched/vstshost.zip" vstshost # Custom package. Will the same work for Win ARM 64? + acquireExternalTool "$CONTAINER_URL/vstsom/m153_47c0856d_adhoc/vstsom.zip" vstsom # Custom package. Will the same work for Win ARM 64? + fi + + acquireExternalTool "$CONTAINER_URL/mingit/${MINGIT_VERSION}/MinGit-${MINGIT_VERSION}-${BIT}-bit.zip" git # Unavailable for Win ARM 64 - https://github.com/git-for-windows/git/releases + acquireExternalTool "$CONTAINER_URL/git-lfs/${LFS_VERSION}/win-arm${BIT}/git-lfs.exe" "git/mingw${BIT}/bin" + acquireExternalTool "$CONTAINER_URL/pdbstr/win-arm${BIT}/1/pdbstr.zip" pdbstr + acquireExternalTool "$CONTAINER_URL/symstore/win-arm${BIT}/1/symstore.zip" symstore + acquireExternalTool "$CONTAINER_URL/vstsom/m153_47c0856d_adhoc/vstsom.zip" tf + acquireExternalTool "$CONTAINER_URL/vswhere/2_8_4/vswhere.zip" vswhere + acquireExternalTool "https://dist.nuget.org/win-x86-commandline/v3.4.4/nuget.exe" nuget + + if [[ "$INCLUDE_NODE6" == "true" ]]; then + acquireExternalTool "${NODE_URL}/v${NODE_VERSION}/${PACKAGERUNTIME}/node.exe" node/bin # Not available for Windows ARM + acquireExternalTool "${NODE_URL}/v${NODE_VERSION}/${PACKAGERUNTIME}/node.lib" node/bin # Not available for Windows ARM + fi + if [[ "$INCLUDE_NODE10" == "true" ]]; then + acquireExternalTool "${NODE_URL}/v${NODE10_VERSION}/${PACKAGERUNTIME}/node.exe" node10/bin # Not available for Windows ARM + acquireExternalTool "${NODE_URL}/v${NODE10_VERSION}/${PACKAGERUNTIME}/node.lib" node10/bin # Not available for Windows ARM + fi + + # Unofficial distribution of Node contains Node 16 for Windows ARM + acquireExternalTool "${NODE_UNOFFICIAL_URL}/v${NODE16_WIN_ARM64_VERSION}/${PACKAGERUNTIME}/node.exe" node16/bin + acquireExternalTool "${NODE_UNOFFICIAL_URL}/v${NODE16_WIN_ARM64_VERSION}/${PACKAGERUNTIME}/node.lib" node16/bin + + # Official distribution of Node contains Node 20 for Windows ARM + acquireExternalTool "${NODE_URL}/v${NODE20_VERSION}/${PACKAGERUNTIME}/node.exe" node20_1/bin + acquireExternalTool "${NODE_URL}/v${NODE20_VERSION}/${PACKAGERUNTIME}/node.lib" node20_1/bin else # Download external tools for Linux and OSX. diff --git a/src/Misc/layoutbin/en-US/strings.json b/src/Misc/layoutbin/en-US/strings.json index 81242bf1f4..0dc72ba647 100644 --- a/src/Misc/layoutbin/en-US/strings.json +++ b/src/Misc/layoutbin/en-US/strings.json @@ -107,6 +107,7 @@ " negotiate (Kerberos or NTLM)", " alt (Basic authentication)", " integrated (Windows default credentials)", + " sp (Service Principal)", " --token Used with --auth pat. Personal access token.", " --userName Used with --auth negotiate or --auth alt. Specify the Windows user", " name in the format: domain\\userName or userName@domain.com", diff --git a/src/Misc/layoutbin/installdependencies.sh b/src/Misc/layoutbin/installdependencies.sh index 4587d4acc9..cd2ed3adef 100644 --- a/src/Misc/layoutbin/installdependencies.sh +++ b/src/Misc/layoutbin/installdependencies.sh @@ -271,7 +271,7 @@ then is_sles=1 fi - if ([[ -n $OSTYPE ]] && ([[ $OSTYPE == *"suse"* ]] || [[$is_sles == 1]])) + if ([[ -n $OSTYPE ]] && ([[ $OSTYPE == *"suse"* ]] || ([[ -n $is_sles ]] && [[ $is_sles == 1 ]]))) then echo "The current OS is SUSE based" command -v zypper @@ -317,6 +317,28 @@ then print_errormessage exit 1 fi + elif [ -e /etc/azurelinux-release ] + then + echo "The current OS is Azure Linux based" + echo "--------Azure Linux Version--------" + cat /etc/azurelinux-release + echo "------------------------------" + + command -v tdnf + if [ $? -eq 0 ] + then + tdnf install -y icu + if [ $? -ne 0 ] + then + echo "'tdnf' failed with exit code '$?'" + print_errormessage + exit 1 + fi + else + echo "Can not find 'tdnf'" + print_errormessage + exit 1 + fi else echo "Can't detect current OS type based on $filepath." print_errormessage diff --git a/src/Test/L0/ConstantGenerationL0.cs b/src/Test/L0/ConstantGenerationL0.cs index 64d2f3a085..9d46b838b1 100644 --- a/src/Test/L0/ConstantGenerationL0.cs +++ b/src/Test/L0/ConstantGenerationL0.cs @@ -17,6 +17,7 @@ public void BuildConstantGenerateSucceed() { "win-x64", "win-x86", + "win-arm64", "linux-x64", "linux-arm", "linux-arm64", diff --git a/src/dev.sh b/src/dev.sh index e8c08ebadc..e4b2834ffe 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -91,6 +91,11 @@ function restore_sdk_and_runtime() { dotnet_windows_dir=${dotnet_windows_dir:0:1}:${dotnet_windows_dir:1} local architecture architecture=$(echo "$RUNTIME_ID" | cut -d "-" -f2) + + # We compile on an x64 machine, even when targeting ARM64. Thereby we are installing the x64 version of .NET instead of the arm64 version. + if [[ "$architecture" == "arm64" ]]; then + architecture="x64" + fi printf "\nInstalling SDK...\n" powershell -NoLogo -Sta -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command "& \"${DOTNET_INSTALL_SCRIPT_PATH}\" -Version ${DOTNET_SDK_VERSION} -InstallDir \"${dotnet_windows_dir}\" -Architecture ${architecture} -NoPath; exit \$LastExitCode;" || checkRC "${DOTNET_INSTALL_SCRIPT_NAME} (SDK)" @@ -113,11 +118,17 @@ function detect_platform_and_runtime_id() { if [[ ($(uname) == "Linux") || ($(uname) == "Darwin") ]]; then CURRENT_PLATFORM=$(uname | awk '{print tolower($0)}') fi - + if [[ "$CURRENT_PLATFORM" == 'windows' ]]; then + local processor_type=$(detect_system_architecture) + echo "Detected Process Arch: $processor_type" + + # Default to win-x64 DETECTED_RUNTIME_ID='win-x64' - if [[ "$PROCESSOR_ARCHITECTURE" == 'x86' ]]; then + if [[ "$processor_type" == 'x86' ]]; then DETECTED_RUNTIME_ID='win-x86' + elif [[ "$processor_type" == 'ARM64' ]]; then + DETECTED_RUNTIME_ID='win-arm64' fi elif [[ "$CURRENT_PLATFORM" == 'linux' ]]; then DETECTED_RUNTIME_ID="linux-x64" @@ -348,6 +359,60 @@ function cmd_report() { fi } +function cmd_lint() { + heading "Linting source code" + + "${DOTNET_DIR}/dotnet" format -v diag "$REPO_ROOT/azure-pipelines-agent.sln" || checkRC "cmd_lint" +} + +function cmd_lint_verify() { + heading "Validating linted code" + + "${DOTNET_DIR}/dotnet" format --verify-no-changes -v diag "$REPO_ROOT/azure-pipelines-agent.sln" || checkRC "cmd_lint_verify" +} + +function detect_system_architecture() { + local processor # Variable to hold the processor type (e.g., x, ARM) + local os_arch # Variable to hold the OS bitness (e.g., 64, 86) + + # Detect processor type using PROCESSOR_IDENTIFIER + # Check for AMD64 or Intel in the variable to classify as "x" (covers x86 and x64 processors) + if [[ "$PROCESSOR_IDENTIFIER" =~ "AMD64" || "$PROCESSOR_IDENTIFIER" =~ "Intel64" ]]; then + processor="x" + # Check for ARM64 in the variable to classify as "ARM" + elif [[ "$PROCESSOR_IDENTIFIER" =~ "ARM" || "$PROCESSOR_IDENTIFIER" =~ "Arm" ]]; then + processor="ARM" + # Default to "x" for unknown or unhandled cases + else + processor="x" + fi + + # Detect OS bitness using uname + # "x86_64" indicates a 64-bit operating system + if [[ "$(uname -m)" == "x86_64" ]]; then + os_arch="64" + # "i686" or "i386" indicates a 32-bit operating system + elif [[ "$(uname -m)" == "i686" || "$(uname -m)" == "i386" ]]; then + os_arch="86" + # "aarch64" indicates a 64-bit ARM operating system + elif [[ "$(uname -m)" == "aarch64" ]]; then + os_arch="64" + # Default to "64" for unknown or unhandled cases + else + os_arch="64" + fi + + # Note: AMD32 does not exist as a specific label; 32-bit AMD processors are referred to as x86. + # ARM32 also does not exist in this context; ARM processors are always 64-bit. + + # Combine processor type and OS bitness for the final result + # Examples: + # - "x64" for Intel/AMD 64-bit + # - "x86" for Intel/AMD 32-bit + # - "ARM64" for ARM 64-bit + echo "${processor}${os_arch}" +} + detect_platform_and_runtime_id echo "Current platform: $CURRENT_PLATFORM" echo "Current runtime ID: $DETECTED_RUNTIME_ID" @@ -358,7 +423,7 @@ else RUNTIME_ID=$DETECTED_RUNTIME_ID fi -_VALID_RIDS='linux-x64:linux-arm:linux-arm64:linux-musl-x64:linux-musl-arm64:osx-x64:osx-arm64:win-x64:win-x86' +_VALID_RIDS='linux-x64:linux-arm:linux-arm64:linux-musl-x64:linux-musl-arm64:osx-x64:osx-arm64:win-x64:win-x86:win-arm64' if [[ ":$_VALID_RIDS:" != *:$RUNTIME_ID:* ]]; then failed "must specify a valid target runtime ID (one of: $_VALID_RIDS)" fi @@ -414,6 +479,8 @@ case $DEV_CMD in "p") cmd_package ;; "hash") cmd_hash ;; "report") cmd_report ;; +"lint") cmd_lint ;; +"lint-verify") cmd_lint_verify ;; *) echo "Invalid command. Use (l)ayout, (b)uild, (t)est, test(l0), test(l1), or (p)ackage." ;; esac diff --git a/src/dir.proj b/src/dir.proj index af6cfc142b..d4368522f4 100644 --- a/src/dir.proj +++ b/src/dir.proj @@ -74,7 +74,7 @@ SkipNonExistentProjects="false" StopOnFirstFailure="true" Properties="Configuration=$(BUILDCONFIG);PackageRuntime=$(PackageRuntime);Version=$(AgentVersion);RuntimeIdentifier=$(PackageRuntime);PublishDir=$(LayoutRoot)/bin;TreatWarningsAsErrors=$(TreatWarningsAsErrors)" /> - + @@ -110,10 +110,10 @@ - - - - + + + +