diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SpecCpuProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SpecCpuProfileTests.cs index 7a01210243..6c4b84230b 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SpecCpuProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/SpecCpuProfileTests.cs @@ -5,6 +5,7 @@ namespace VirtualClient.Actions { using System; using System.Collections.Generic; + using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Threading; @@ -45,6 +46,59 @@ public void SpecCpuWorkloadProfileParametersAreInlinedCorrectly(string profile) } } + [Test] + [TestCase("PERF-SPECCPU-INTRATE.json", PlatformID.Unix, "intrate")] + [TestCase("PERF-SPECCPU-INTRATE.json", PlatformID.Win32NT, "505 525 541 548 557")] + [TestCase("PERF-SPECCPU-FPRATE.json", PlatformID.Unix, "fprate")] + [TestCase("PERF-SPECCPU-FPRATE.json", PlatformID.Win32NT, "503 508 510 519 544 549 554")] + public async Task SpecCpuRateProfilesRunTheFullSuiteWithNumaProcessorAffinityDisabledByDefault( + string profileName, + PlatformID platform, + string expectedBenchmarks) + { + this.mockFixture.Setup(platform); + + ExecutionProfile profile = await ExecutionProfile.ReadProfileAsync(Path.Combine(MockFixture.TestResourcesDirectory, "profiles", profileName)); + profile.Parameters["CopyCount"] = 4; + + Assert.AreEqual(1, profile.Actions.Count, "A SPEC CPU rate suite must run as one action, not separate benchmarks."); + ExecutionProfileElement action = profile.Actions.Single(); + Assert.AreEqual("SpecCpuExecutor", action.Type); + Assert.AreEqual("ExecuteSPECBenchmark", action.Parameters["Scenario"]); + Assert.AreEqual("$.Parameters.Benchmarks", action.Parameters["Benchmarks"], "The suite action must use the platform-specific benchmark selection."); + Assert.AreEqual(string.Empty, profile.Parameters["NumaProcessorAffinity"]); + Assert.AreEqual("$.Parameters.NumaProcessorAffinity", action.Parameters["NumaProcessorAffinity"]); + Assert.IsFalse(profile.Actions.Any(item => item.Parameters["Scenario"].ToString().StartsWith("benchmark_", StringComparison.OrdinalIgnoreCase))); + + profile.Inline(); + await ProfileExpressionEvaluator.Instance.EvaluateAsync(this.mockFixture.Dependencies, action.Parameters); + + Assert.AreEqual(expectedBenchmarks, action.Parameters["Benchmarks"]); + Assert.AreEqual(string.Empty, action.Parameters["NumaProcessorAffinity"]); + Assert.IsFalse(action.Parameters.ContainsKey("EnableNumaProcessorAffinitySymmetric")); + Assert.AreEqual("4", action.Parameters["CopyCount"].ToString()); + } + + [Test] + [TestCase("PERF-SPECCPU-INTRATE.json", 2, "symmetric")] + [TestCase("PERF-SPECCPU-INTRATE.json", 4, "SyMmEtRiC")] + [TestCase("PERF-SPECCPU-FPRATE.json", 2, "symmetric")] + [TestCase("PERF-SPECCPU-FPRATE.json", 4, "0,1,48,49")] + public async Task SpecCpuRateProfilesForwardNumaProcessorAffinity(string profileName, int copyCount, string affinity) + { + this.mockFixture.Setup(PlatformID.Unix); + ExecutionProfile profile = await ExecutionProfile.ReadProfileAsync(Path.Combine(MockFixture.TestResourcesDirectory, "profiles", profileName)); + profile.Parameters["CopyCount"] = copyCount; + profile.Parameters["NumaProcessorAffinity"] = affinity; + + profile.Inline(); + + ExecutionProfileElement action = profile.Actions.Single(); + Assert.AreEqual(affinity, action.Parameters["NumaProcessorAffinity"]); + Assert.IsFalse(action.Parameters.ContainsKey("EnableNumaProcessorAffinitySymmetric")); + Assert.AreEqual(copyCount.ToString(), action.Parameters["CopyCount"].ToString()); + } + [Test] [TestCase("PERF-SPECCPU2026-FPRATE.json", Architecture.X64)] [TestCase("PERF-SPECCPU2026-FPRATE.json", Architecture.Arm64)] diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpu2017ExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpu2017ExecutorTests.cs index f26f41d8e8..f33e5e09e4 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpu2017ExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpu2017ExecutorTests.cs @@ -10,6 +10,7 @@ namespace VirtualClient.Actions using System.IO.Abstractions; using System.Linq; using System.Reflection; + using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -589,10 +590,349 @@ public async Task SpecCpu2017ExecutorDoesNotApplyGcc15WorkaroundWhenGccVersionIs Assert.IsFalse(writtenConfigText.Contains("$Gcc10Workaround$"), "Placeholder '$Gcc10Workaround$' should be replaced."); } - private void SetupLinux() + [TestCase("intrate", "0-3", "$SPECCOPYNUM", null)] + [TestCase("intrate", "0,1,48,49", "$BIND", "bind = 0,1,48,49")] + [TestCase("intrate", "2,4,8,10", "$BIND", "bind = 2,4,8,10")] + [TestCase("fprate", "2,4,8,10", "$BIND", "bind = 2,4,8,10")] + public async Task SpecCpu2017ExecutorPinsRateCopies(string profile, string cpuIds, string copyCpu, string expectedBind) + { + this.SetupLinux(); + this.SetupInstalledAffinityRun(cpuIds); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.SpecProfile)] = profile; + this.mockFixture.Parameters[nameof(SpecCpuExecutor.Benchmarks)] = profile; + + string pinnedConfig = null; + string pinnedPath = null; + this.mockFixture.File.Setup(f => f.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, content, token) => + { + pinnedPath = path; + pinnedConfig = content; + }) + .Returns(Task.CompletedTask); + + List commands = new List(); + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => + { + commands.Add($"{exe} {arguments}"); + return new InMemoryProcess + { + StartInfo = new ProcessStartInfo { FileName = exe, Arguments = arguments }, + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + }; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + } + + StringAssert.Contains("vc-linux-x64-2017-affinity-", Path.GetFileName(pinnedPath)); + StringAssert.Contains($"submit = numactl --localalloc --physcpubind={copyCpu} -- $command", pinnedConfig); + if (expectedBind != null) + { + StringAssert.Contains(expectedBind, pinnedConfig); + } + + Assert.IsTrue(commands.Any(command => command == "numactl --version")); + Assert.IsTrue(commands.Any(command => command == $"numactl --localalloc -C {cpuIds} true")); + Assert.IsTrue(commands.Any(command => command.Contains($"--config {Path.GetFileName(pinnedPath)}") && command.Contains("--copies 4"))); + } + + [TestCase(2, 2, "0,48", "0,48", null)] + [TestCase(2, 3, "0,1,48", "0,1,48", null)] + [TestCase(2, 4, "0,1,48,49", "0,1,48,49", null)] + [TestCase(2, 40, "0-19,48-67", "0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67", null)] + [TestCase(3, 2, "0,32", "0,32", null)] + [TestCase(3, 3, "0,32,64", "0,32,64", null)] + [TestCase(3, 6, "0,1,32,33,64,65", "0,1,32,33,64,65", null)] + [TestCase(3, 8, "0-2,32-34,64,65", "0,1,2,32,33,34,64,65", null)] + [TestCase(3, 12, "0-3,32-35,64-67", "0,1,2,3,32,33,34,35,64,65,66,67", null)] + [TestCase(3, 13, "0-4,32-35,64-67", "0,1,2,3,4,32,33,34,35,64,65,66,67", null)] + [TestCase(3, 14, "0-4,32-36,64-67", "0,1,2,3,4,32,33,34,35,36,64,65,66,67", null)] + [TestCase(3, 40, "0-13,32-44,64-76", "0,1,2,3,4,5,6,7,8,9,10,11,12,13,32,33,34,35,36,37,38,39,40,41,42,43,44,64,65,66,67,68,69,70,71,72,73,74,75,76", null)] + [TestCase(2, 2, "0,48", "0,48", "symmetric")] + [TestCase(2, 4, "0,1,48,49", "0,1,48,49", "SyMmEtRiC")] + public async Task SpecCpu2017ExecutorSelectsSymmetricNumaAffinity(int nodeCount, int copyCount, string expectedCoreSpec, string expectedCpus, string affinity) + { + this.SetupLinux(Architecture.Arm64); + this.SetupInstalledAffinityRun(affinity ?? string.Empty); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.EnableNumaProcessorAffinitySymmetric)] = affinity == null; + this.mockFixture.Parameters[nameof(SpecCpuExecutor.CopyCount)] = copyCount; + + int coresPerNode = 96 / nodeCount; + string topology = $"available: {nodeCount} nodes (0-{nodeCount - 1})\n" + string.Join("\n", Enumerable.Range(0, nodeCount).Select(node => + $"node {node} cpus: {string.Join(" ", Enumerable.Range(node * coresPerNode, coresPerNode))}\nnode {node} size: 16000 MB\nnode {node} free: 12000 MB")); + + string pinnedConfig = null; + this.mockFixture.File.Setup(f => f.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, content, token) => pinnedConfig = content) + .Returns(Task.CompletedTask); + + List commands = new List(); + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => + { + commands.Add($"{exe} {arguments}"); + return new InMemoryProcess + { + StartInfo = new ProcessStartInfo { FileName = exe, Arguments = arguments }, + StandardOutput = new ConcurrentBuffer(new StringBuilder(exe == "numactl" && arguments == "--hardware" ? topology : string.Empty)), + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + }; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.IsTrue(executor.EnableNumaProcessorAffinitySymmetric); + await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + Assert.IsTrue(executor.EnableNumaProcessorAffinitySymmetric); + } + + StringAssert.Contains($"bind = {expectedCpus}", pinnedConfig); + StringAssert.Contains("submit = numactl --localalloc --physcpubind=$BIND -- $command", pinnedConfig); + Assert.IsTrue(commands.Any(command => command == $"numactl --localalloc -C {expectedCoreSpec} true")); + Assert.IsTrue(commands.Any(command => command.Contains($"--copies {copyCount}"))); + EventContext selection = (EventContext)this.mockFixture.Logger.Single(entry => entry.Item2.Name.EndsWith(".SymmetricAffinitySelected", StringComparison.Ordinal)).Item3; + Assert.AreEqual(expectedCpus, selection.Properties["cpuIds"]); + Assert.AreEqual(true, selection.Properties["pinningEnabled"]); + Assert.AreEqual(expectedCpus, ((IDictionary)selection.Properties["metadata_scenario"])["NumaProcessorAffinity"]); + } + + [Test] + public void SpecCpu2017ExecutorRejectsManualAndSymmetricAffinityTogether() + { + this.SetupLinux(); + this.SetupInstalledAffinityRun("0-3"); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.EnableNumaProcessorAffinitySymmetric)] = true; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(async () => await executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void SpecCpu2017ExecutorRejectsSymmetricAffinityWhenTopologyIsUnreadable() + { + this.SetupLinux(); + this.SetupInstalledAffinityRun(string.Empty); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.EnableNumaProcessorAffinitySymmetric)] = true; + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => new InMemoryProcess + { + StartInfo = new ProcessStartInfo { FileName = exe, Arguments = arguments }, + StandardOutput = new ConcurrentBuffer(new StringBuilder("invalid topology")), + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(async () => await executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void SpecCpu2017ExecutorRejectsSymmetricAffinityOnSpeedRuns() + { + this.SetupLinux(); + this.SetupInstalledAffinityRun(string.Empty); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.EnableNumaProcessorAffinitySymmetric)] = true; + this.mockFixture.Parameters[nameof(SpecCpuExecutor.SpecProfile)] = "intspeed"; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(async () => await executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void SpecCpu2017ExecutorRejectsSymmetricAffinityWhenTopologyHasTooFewCpus() + { + this.SetupLinux(); + this.SetupInstalledAffinityRun(string.Empty); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.EnableNumaProcessorAffinitySymmetric)] = true; + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => new InMemoryProcess + { + StartInfo = new ProcessStartInfo { FileName = exe, Arguments = arguments }, + StandardOutput = new ConcurrentBuffer(new StringBuilder("available: 1 nodes (0)\nnode 0 cpus: 0 1\nnode 0 size: 16000 MB\nnode 0 free: 12000 MB")), + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(async () => await executor.ExecuteAsync(CancellationToken.None)); + } + } + + [TestCase("", false, Architecture.X64)] + [TestCase("0-3", true, Architecture.X64)] + [TestCase("0-3", true, Architecture.Arm64)] + public async Task SpecCpu2017ExecutorPreservesSuiteTelemetryAndMetadataWithOptionalAffinity(string numaProcessorAffinity, bool pinned, Architecture architecture) + { + this.SetupLinux(architecture); + this.SetupInstalledAffinityRun(numaProcessorAffinity); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.Scenario)] = "ExecuteSPECBenchmark"; + string topology = "available: 2 nodes (0-1)\nnode 0 cpus: 0 1\nnode 0 size: 16000 MB\nnode 0 free: 12000 MB\nnode 1 cpus: 2 3\nnode 1 size: 16000 MB\nnode 1 free: 11000 MB"; + + string configurationFile = null; + this.mockFixture.File.Setup(f => f.WriteAllTextAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((path, content, token) => configurationFile = Path.GetFileName(path)) + .Returns(Task.CompletedTask); + + List commands = new List(); + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => + { + commands.Add($"{exe} {arguments}"); + return new InMemoryProcess + { + StartInfo = new ProcessStartInfo { FileName = exe, Arguments = arguments }, + StandardOutput = new ConcurrentBuffer(new StringBuilder(exe == "numactl" && arguments == "--hardware" ? topology : string.Empty)), + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + }; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + } + + string expectedConfig = pinned ? configurationFile : $"vc-linux-{(architecture == Architecture.Arm64 ? "arm64" : "x64")}-2017.cfg"; + Assert.AreEqual(pinned, commands.Any(command => command == "numactl --version")); + Assert.AreEqual(pinned, commands.Any(command => command == "numactl --hardware")); + Assert.AreEqual(1, commands.Count(command => command.StartsWith("bash runspeccpu.sh", StringComparison.Ordinal))); + Assert.IsTrue(commands.Any(command => command.Contains($"--config {expectedConfig}") && command.EndsWith("--reportable intrate\""))); + + var topologyEvents = this.mockFixture.Logger.Where(entry => entry.Item2.Name.EndsWith(".NumactlTopology", StringComparison.Ordinal)).ToList(); + Assert.AreEqual(pinned ? 1 : 0, topologyEvents.Count); + if (pinned) + { + EventContext topologyContext = (EventContext)topologyEvents.Single().Item3; + Assert.AreEqual("0,1,2,3", topologyContext.Properties["cpuIds"]); + Assert.AreEqual(true, topologyContext.Properties["available"]); + Assert.AreEqual(topology, topologyContext.Properties["topology"]); + } + + EventContext runContext = (EventContext)this.mockFixture.Logger.Single(entry => entry.Item2.Name.EndsWith(".RunStarted", StringComparison.Ordinal)).Item3; + Assert.AreEqual("ExecuteSPECBenchmark", runContext.Properties["scenario"]); + Assert.AreEqual(pinned, runContext.Properties["pinningEnabled"]); + Assert.AreEqual(expectedConfig, runContext.Properties["configurationFile"]); + + EventContext metricsContext = (EventContext)this.mockFixture.Logger.Single(entry => entry.Item2.Name.EndsWith(".MetricsCaptured", StringComparison.Ordinal)).Item3; + IDictionary metadata = (IDictionary)metricsContext.Properties["metadata_scenario"]; + Assert.AreEqual("SPECcpu/intrate", metadata["toolName"]); + StringAssert.Contains($"--config {expectedConfig}", metadata["toolArguments"].ToString()); + } + + [TestCase("", 1)] + [TestCase("available: 2 nodes (0-1)\nnode 0 cpus: 0 1\nnode 0 size: 16000 MB\nnode 0 free: 12000 MB\nnode 1 cpus: 2 3\nnode 1 size: 16000 MB", 0)] + [TestCase("available: 1 nodes (0)\nnode 0 cpus: 8 9 10 11\nnode 0 size: 16000 MB\nnode 0 free: 12000 MB", 0)] + public async Task SpecCpu2017ExecutorReportsUnavailableTopologyWithoutBlockingTheRun(string topology, int probeExitCode) + { + this.SetupLinux(Architecture.Arm64); + this.SetupInstalledAffinityRun("0-3"); + bool ranSuite = false; + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => + { + ranSuite |= exe == "bash"; + return new InMemoryProcess + { + StartInfo = new ProcessStartInfo { FileName = exe, Arguments = arguments }, + StandardOutput = new ConcurrentBuffer(new StringBuilder(exe == "numactl" && arguments == "--hardware" ? topology : string.Empty)), + ExitCode = exe == "numactl" && arguments == "--hardware" ? probeExitCode : 0, + OnStart = () => true, + OnHasExited = () => true + }; + }; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + } + + EventContext topologyContext = (EventContext)this.mockFixture.Logger.Single(entry => entry.Item2.Name.EndsWith(".NumactlTopology", StringComparison.Ordinal)).Item3; + Assert.AreEqual(false, topologyContext.Properties["available"]); + Assert.IsFalse(this.mockFixture.Logger.Any(entry => entry.Item2.Name == "SPECcpu.ScenarioResult")); + Assert.IsTrue(ranSuite); + } + + [TestCase("0-2")] + [TestCase("0,invalid,2,3")] + public void SpecCpu2017ExecutorRejectsInvalidNumaProcessorAffinity(string cpuIds) + { + this.SetupLinux(); + this.SetupInstalledAffinityRun(cpuIds); + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(async () => await executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void SpecCpu2017ExecutorRejectsAffinityOnSpeedRuns() + { + this.SetupLinux(); + this.SetupInstalledAffinityRun("0-3"); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.SpecProfile)] = "intspeed"; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(async () => await executor.ExecuteAsync(CancellationToken.None)); + } + } + + [TestCase("--version", typeof(DependencyException))] + [TestCase("--localalloc", typeof(ArgumentException))] + public void SpecCpu2017ExecutorValidatesNumactl(string failingCommand, Type expectedException) + { + this.SetupLinux(); + this.SetupInstalledAffinityRun("0-3"); + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => new InMemoryProcess + { + StartInfo = new ProcessStartInfo { FileName = exe, Arguments = arguments }, + ExitCode = exe == "numactl" && arguments.StartsWith(failingCommand) ? 1 : 0, + OnStart = () => true, + OnHasExited = () => true + }; + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(expectedException, async () => await executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void SpecCpu2017ExecutorRejectsAffinityOnWindows() + { + this.SetupWindows(); + this.SetupInstalledAffinityRun("0-3"); + + using (TestSpecCpu2017Executor executor = new TestSpecCpu2017Executor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(async () => await executor.ExecuteAsync(CancellationToken.None)); + } + } + + private void SetupInstalledAffinityRun(string cpuIds) + { + State state = new State(new Dictionary { ["SpecCpuInitialized"] = true }); + this.mockFixture.StateManager.OnGetState().ReturnsAsync(JObject.Parse(state.ToJson())); + this.mockFixture.Parameters[nameof(SpecCpuExecutor.NumaProcessorAffinity)] = cpuIds; + } + + private void SetupLinux(Architecture architecture = Architecture.X64) { this.mockFixture = new MockFixture(); - this.mockFixture.Setup(PlatformID.Unix); + this.mockFixture.Setup(PlatformID.Unix, architecture); this.mockPackage = new DependencyPath("SPECcpu", this.mockFixture.PlatformSpecifics.GetPackagePath("speccpu", "1.1.8")); this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(this.mockPackage); diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs index acfa03c6dc..2fa904b485 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs @@ -1,4 +1,5 @@ -// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + // Licensed under the MIT License. namespace VirtualClient.Actions diff --git a/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs b/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs index cc73b22d66..ed8d029298 100644 --- a/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs @@ -5,18 +5,24 @@ namespace VirtualClient.Actions { using System; using System.Collections.Generic; + using System.ComponentModel; + using System.Globalization; using System.IO; using System.IO.Abstractions; using System.Linq; using System.Runtime.InteropServices; + using System.Security.Cryptography; using System.Text; + using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using global::VirtualClient.Common; using global::VirtualClient.Common.Extensions; + using global::VirtualClient.Common.ProcessAffinity; using global::VirtualClient.Common.Telemetry; using global::VirtualClient.Contracts; using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; using VirtualClient.Contracts.Metadata; /// @@ -153,8 +159,42 @@ public int CopyCount } } + /// + /// Optional list of physical CPU IDs to assign to SPECrate copies, one CPU per copy. + /// Set to "symmetric" (case-insensitive) to select CPUs evenly across NUMA nodes automatically. + /// For two 48-CPU NUMA nodes (CPUs 0-47 and 48-95): + /// 2 copies: NumaProcessorAffinity=0,48; + /// 4 copies: NumaProcessorAffinity=0,1,48,49; + /// 40 copies: NumaProcessorAffinity=0-19,48-67. + /// For three 32-CPU NUMA nodes (CPUs 0-31, 32-63, and 64-95): + /// 3 copies: NumaProcessorAffinity=0,32,64; + /// 40 copies: NumaProcessorAffinity=0-13,32-44,64-76. + /// + public string NumaProcessorAffinity + { + get + { + return this.Parameters.GetValue(nameof(SpecCpuExecutor.NumaProcessorAffinity), string.Empty); + } + } + + /// + /// When enabled, distributes SPECrate copies as evenly as available CPUs allow across NUMA nodes, + /// assigning one CPU per copy using numactl. + /// Defaults to false unless NumaProcessorAffinity is "symmetric"; cannot be combined with manual CPU IDs. + /// + public bool EnableNumaProcessorAffinitySymmetric + { + get + { + return string.Equals(this.NumaProcessorAffinity, "symmetric", StringComparison.OrdinalIgnoreCase) + || this.Parameters.GetValue(nameof(SpecCpuExecutor.EnableNumaProcessorAffinitySymmetric), false); + } + } + /// /// A feature flag to apply. For example 'UseCsvResults' can be used to parse the CSV file results vs. the standard output. + /// The CSV results have finer-grained results with additional significant figures. /// public string FeatureFlag { @@ -216,11 +256,32 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel /// A token that can be used to cancel the operation. protected async Task ExecuteSpecCpuAsync(EventContext telemetryContext, CancellationToken cancellationToken) { + string stage = "Configuration"; try { using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) { - string commandLineArguments = this.GetCommandLineArguments(); + if (this.EnableNumaProcessorAffinitySymmetric) + { + stage = "SymmetricAffinityConfiguration"; + this.Parameters[nameof(this.EnableNumaProcessorAffinitySymmetric)] = true; + string affinity = await this.GetSymmetricNumaProcessorAffinityAsync(cancellationToken); + this.Parameters[nameof(this.NumaProcessorAffinity)] = affinity; + this.MetadataContract.Add(nameof(this.NumaProcessorAffinity), affinity, MetadataContract.ScenarioCategory); + this.MetadataContract.Apply(telemetryContext); + this.LogSpecCpuStep(telemetryContext, "SymmetricAffinitySelected", ("cpuIds", affinity)); + } + + string configurationFile = this.GetConfigurationFileName(); + if (!string.IsNullOrWhiteSpace(this.NumaProcessorAffinity)) + { + stage = "AffinityConfiguration"; + configurationFile = await this.WritePinnedSpecCpuConfigAsync(telemetryContext, cancellationToken); + } + + this.LogSpecCpuStep(telemetryContext, "ConfigSelected", ("configurationFile", configurationFile)); + string commandLineArguments = this.GetCommandLineArguments(configurationFile); + string command; string commandArguments; @@ -235,6 +296,9 @@ protected async Task ExecuteSpecCpuAsync(EventContext telemetryContext, Cancella commandArguments = $"/c {SpecCpuExecutor.SpecCpuRunBat} {commandLineArguments}"; } + stage = "Run"; + this.LogSpecCpuStep(telemetryContext, "RunStarted", ("configurationFile", configurationFile), ("command", command), ("arguments", commandArguments)); + DateTime runStart = DateTime.UtcNow; using (IProcessProxy process = await this.ExecuteCommandAsync( command, commandArguments, @@ -246,13 +310,27 @@ protected async Task ExecuteSpecCpuAsync(EventContext telemetryContext, Cancella if (!cancellationToken.IsCancellationRequested) { await this.LogProcessDetailsAsync(process, telemetryContext, "SPECcpu", logToFile: true); + this.LogSpecCpuStep(telemetryContext, "RunCompleted", ("configurationFile", configurationFile), ("exitCode", process.ExitCode), ("durationMs", (DateTime.UtcNow - runStart).TotalMilliseconds)); process.ThrowIfWorkloadFailed(); + stage = "Metrics"; + this.MetadataContract.AddForScenario($"SPECcpu/{this.SpecProfile}", process.FullCommand(), toolVersion: null); + this.MetadataContract.Apply(telemetryContext); await this.CaptureMetricsAsync(process, commandLineArguments, telemetryContext, cancellationToken); } } } } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + this.LogSpecCpuStep(telemetryContext, "Cancelled", ("stage", stage)); + throw; + } + catch (Exception exc) + { + this.LogSpecCpuFailure(telemetryContext, stage, exc); + throw; + } finally { if (!cancellationToken.IsCancellationRequested) @@ -269,28 +347,48 @@ protected async Task ExecuteSpecCpuAsync(EventContext telemetryContext, Cancella /// A token that can be used to cancel the operation. protected async Task InitializeSpecCpuAsync(EventContext telemetryContext, CancellationToken cancellationToken) { - DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, CancellationToken.None); - - if (workloadPackage == null) + string stage = "PackageLookup"; + try { - throw new DependencyException( - $"The expected package '{this.PackageName}' does not exist on the system or is not registered.", - ErrorReason.WorkloadDependencyMissing); - } + this.LogSpecCpuStep(telemetryContext, "PackageLookupStarted", ("packageName", this.PackageName)); + DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, CancellationToken.None); - this.PackageDirectory = workloadPackage.Path; - this.ResultsDirectory = this.Combine(this.PackageDirectory, "result"); + if (workloadPackage == null) + { + throw new DependencyException( + $"The expected package '{this.PackageName}' does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } - if (this.fileSystem.Directory.Exists(this.ResultsDirectory)) - { - await this.fileSystem.Directory.DeleteAsync(this.ResultsDirectory); - this.fileSystem.Directory.CreateDirectory(this.ResultsDirectory); - } + this.PackageDirectory = workloadPackage.Path; + this.ResultsDirectory = this.Combine(this.PackageDirectory, "result"); + this.LogSpecCpuStep(telemetryContext, "PackageReady", ("packageName", this.PackageName)); + + // Clean any previous results from the system. + if (this.fileSystem.Directory.Exists(this.ResultsDirectory)) + { + await this.fileSystem.Directory.DeleteAsync(this.ResultsDirectory); + this.fileSystem.Directory.CreateDirectory(this.ResultsDirectory); + } - string imageFile = this.GetIsoFilePath(workloadPackage); - telemetryContext.AddContext(nameof(imageFile), imageFile); + stage = "Setup"; + string imageFile = this.GetIsoFilePath(workloadPackage); + telemetryContext.AddContext(nameof(imageFile), imageFile); - await this.SetupSpecCpuAsync(imageFile, telemetryContext, cancellationToken); + this.LogSpecCpuStep(telemetryContext, "SetupStarted", ("imageFile", Path.GetFileName(imageFile))); + await this.SetupSpecCpuAsync(imageFile, telemetryContext, cancellationToken); + this.LogSpecCpuStep(telemetryContext, "SetupCompleted"); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + this.LogSpecCpuStep(telemetryContext, "Cancelled", ("stage", stage)); + throw; + } + catch (Exception exc) + { + this.LogSpecCpuFailure(telemetryContext, stage, exc); + throw; + } } /// @@ -439,11 +537,13 @@ private async Task SetupSpecCpuAsync(string isoFilePath, EventContext telemetryC SpecCpuState state = await this.stateManager.GetStateAsync(stateId, cancellationToken) ?? new SpecCpuState(); + this.LogSpecCpuStep(telemetryContext, "SetupStateRead", ("alreadyInitialized", state.SpecCpuInitialized)); if (state.SpecCpuInitialized) { return; } + this.LogSpecCpuStep(telemetryContext, "InstallationStarted"); string mountPath = this.PlatformSpecifics.Combine(this.PlatformSpecifics.GetPackagePath(), "speccpu_mount"); this.fileSystem.Directory.CreateDirectory(mountPath); @@ -458,6 +558,7 @@ private async Task SetupSpecCpuAsync(string isoFilePath, EventContext telemetryC state.SpecCpuInitialized = true; await this.stateManager.SaveStateAsync(stateId, state, cancellationToken); + this.LogSpecCpuStep(telemetryContext, "InstallationCompleted"); } private async Task LinuxSetupAsync( @@ -565,13 +666,6 @@ private async Task CaptureMetricsAsync( { if (!cancellationToken.IsCancellationRequested) { - this.MetadataContract.AddForScenario( - $"SPECcpu/{this.SpecProfile}", - process.FullCommand(), - toolVersion: null); - - this.MetadataContract.Apply(telemetryContext); - bool useCsv = string.Equals(this.FeatureFlag, SpecCpuExecutor.FeatureFlagUseCsvResults, StringComparison.OrdinalIgnoreCase); string extension = useCsv ? "csv" : "txt"; string[] outputFiles = this.fileSystem.Directory.GetFiles( @@ -579,6 +673,7 @@ private async Task CaptureMetricsAsync( this.GetResultsFileSearchPattern(extension), SearchOption.TopDirectoryOnly); + int metricCount = 0; foreach (string file in outputFiles) { KeyValuePair results = await this.LoadResultsAsync(file, cancellationToken); @@ -588,6 +683,7 @@ private async Task CaptureMetricsAsync( IList metrics = parser.Parse(); metrics.LogConsole(this.Scenario, "SPECcpu"); + metricCount += metrics.Count; metrics.ToList().ForEach(m => m.Categorization = $"{this.SpecProfile}-{this.tuning}"); foreach (Metric metric in metrics) @@ -603,6 +699,8 @@ private async Task CaptureMetricsAsync( this.Tags); } } + + this.LogSpecCpuStep(telemetryContext, "MetricsCaptured", ("resultFileCount", outputFiles.Length), ("metricCount", metricCount), ("format", useCsv ? "csv" : "txt")); } } @@ -640,10 +738,9 @@ private async Task RequestLogUploadsAsync(CancellationToken cancellationToken) } } - private string GetCommandLineArguments() + private string GetCommandLineArguments(string configurationFile) { List suites = new List { "intrate", "intspeed", "fprate", "fpspeed" }; - string configurationFile = this.GetConfigurationFileName(); string command = $"--config {configurationFile} --iterations {this.IterationCount} --copies {this.CopyCount} --threads {this.ThreadCount} --tune {this.tuning}"; bool reportable = this.Platform == PlatformID.Unix @@ -654,6 +751,304 @@ private string GetCommandLineArguments() return $"{command} {this.Benchmarks}"; } + /// + /// Reads numactl --hardware and takes the lowest CPU IDs from each node in round-robin order, + /// distributing copies as evenly as node capacity allows. Returns a comma-separated list + /// containing one CPU per . + /// + /// + /// For two 48-CPU nodes (0-47, 48-95), 2 copies select 0 and 48; 4 copies select 0, 1, 48, 49; + /// and 40 copies select 0-19 and 48-67. For three 32-CPU nodes (0-31, 32-63, 64-95), + /// 3 copies select 0, 32, 64 and 40 copies select 0-13, 32-44, 64-76. + /// The returned list contains individual CPU IDs, not range notation. + /// + /// A token used to cancel the topology query. + /// The selected CPU IDs in ascending order, separated by commas. + private async Task GetSymmetricNumaProcessorAffinityAsync(CancellationToken cancellationToken) + { + if (this.Platform != PlatformID.Unix || + (!string.Equals(this.SpecProfile, "intrate", StringComparison.OrdinalIgnoreCase) && + !string.Equals(this.SpecProfile, "fprate", StringComparison.OrdinalIgnoreCase))) + { + throw new NotSupportedException("Symmetric NUMA processor affinity is supported only for Linux intrate and fprate runs."); + } + + if (!string.IsNullOrWhiteSpace(this.NumaProcessorAffinity) && + !string.Equals(this.NumaProcessorAffinity, "symmetric", StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException("NumaProcessorAffinity and EnableNumaProcessorAffinitySymmetric cannot both be specified.", nameof(this.NumaProcessorAffinity)); + } + + if (this.CopyCount <= 0) + { + throw new ArgumentOutOfRangeException(nameof(this.CopyCount), this.CopyCount, "CopyCount must be greater than zero for symmetric NUMA processor affinity."); + } + + IReadOnlyList nodes; + try + { + using (IProcessProxy process = this.systemManager.ProcessManager.CreateProcess("numactl", "--hardware", this.PackageDirectory)) + { + await process.StartAndWaitAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + + string topology = process.StandardOutput.ToString().Trim(); + if (process.IsErrored() || !SpecCpuExecutor.TryParseNumaTopology(topology, Array.Empty(), out nodes)) + { + throw new DependencyException("Symmetric NUMA processor affinity requires a readable numactl --hardware topology.", ErrorReason.WorkloadDependencyMissing); + } + } + } + catch (Win32Exception exc) + { + throw new DependencyException("Symmetric NUMA processor affinity requires numactl to be installed.", exc, ErrorReason.WorkloadDependencyMissing); + } + + List> nodeCpus = nodes + .Where(node => node.Cpus.Count > 0) + .Select(node => (IReadOnlyList)node.Cpus.OrderBy(cpu => cpu).ToList()) + .ToList(); + + if (nodeCpus.Sum(cpus => cpus.Count) < this.CopyCount) + { + throw new ArgumentException($"CopyCount={this.CopyCount} exceeds the number of CPUs available across NUMA nodes.", nameof(this.CopyCount)); + } + + List selectedCpus = new List(this.CopyCount); + for (int index = 0; selectedCpus.Count < this.CopyCount; index++) + { + foreach (IReadOnlyList cpus in nodeCpus) + { + if (index < cpus.Count) + { + selectedCpus.Add(cpus[index]); + if (selectedCpus.Count == this.CopyCount) + { + break; + } + } + } + } + + return string.Join(",", selectedCpus.OrderBy(cpu => cpu)); + } + + private async Task WritePinnedSpecCpuConfigAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.LogSpecCpuStep(telemetryContext, "AffinityValidationStarted", ("requestedCpuIds", this.NumaProcessorAffinity)); + if (this.Platform != PlatformID.Unix || + (!string.Equals(this.SpecProfile, "intrate", StringComparison.OrdinalIgnoreCase) && + !string.Equals(this.SpecProfile, "fprate", StringComparison.OrdinalIgnoreCase))) + { + throw new NotSupportedException("SPEC CPU per-copy processor affinity is supported only for Linux intrate and fprate runs."); + } + + LinuxProcessAffinityConfiguration affinityConfig = (LinuxProcessAffinityConfiguration)ProcessAffinityConfiguration.Create(this.Platform, this.NumaProcessorAffinity); + IReadOnlyList cores = affinityConfig.Cores; + if (cores.Count != this.CopyCount) + { + throw new ArgumentException($"NumaProcessorAffinity must specify exactly one distinct CPU per SPEC copy (CopyCount={this.CopyCount}, CPUs={cores.Count}).", nameof(this.NumaProcessorAffinity)); + } + + bool useCopyNumber = cores.Select((core, copy) => core == copy).All(matches => matches); + string copyCpu = useCopyNumber ? "$SPECCOPYNUM" : "$BIND"; + this.LogSpecCpuStep(telemetryContext, "AffinityValidated", ("cpuIds", string.Join(",", cores)), ("mapping", copyCpu)); + + this.LogSpecCpuStep(telemetryContext, "NumactlCheckStarted"); + try + { + using (IProcessProxy process = this.systemManager.ProcessManager.CreateProcess("numactl", "--version", this.PackageDirectory)) + { + await process.StartAndWaitAsync(cancellationToken); + if (!cancellationToken.IsCancellationRequested && process.IsErrored()) + { + throw new DependencyException("SPEC CPU processor affinity requires a working numactl installation.", ErrorReason.WorkloadDependencyMissing); + } + + this.LogSpecCpuStep(telemetryContext, "NumactlChecked", ("exitCode", process.ExitCode)); + } + } + catch (Win32Exception exc) + { + throw new DependencyException("SPEC CPU processor affinity requires numactl to be installed.", exc, ErrorReason.WorkloadDependencyMissing); + } + + try + { + using (IProcessProxy process = this.systemManager.ProcessManager.CreateProcess("numactl", "--hardware", this.PackageDirectory)) + { + await process.StartAndWaitAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + + string topology = process.StandardOutput.ToString().Trim(); + bool available = !process.IsErrored() && SpecCpuExecutor.TryParseNumaTopology(topology, cores, out _); + + this.LogSpecCpuStep( + telemetryContext, + "NumactlTopology", + ("cpuIds", string.Join(",", cores)), + ("topology", topology), + ("available", available), + ("exitCode", process.ExitCode), + ("error", process.IsErrored() ? process.StandardError.ToString().Trim() : available ? string.Empty : "Unable to parse NUMA topology for the selected CPUs.")); + } + } + catch (Win32Exception exc) + { + this.LogSpecCpuStep(telemetryContext, "NumactlTopology", ("cpuIds", string.Join(",", cores)), ("available", false), ("error", exc.Message)); + } + + (string executable, string arguments) = affinityConfig.GetAffinityProcessInfo("true"); + this.LogSpecCpuStep(telemetryContext, "AffinityPreflightStarted", ("arguments", $"--localalloc {arguments}")); + using (IProcessProxy process = this.systemManager.ProcessManager.CreateProcess(executable, $"--localalloc {arguments}", this.PackageDirectory)) + { + await process.StartAndWaitAsync(cancellationToken); + if (!cancellationToken.IsCancellationRequested && process.IsErrored()) + { + throw new ArgumentException($"NumaProcessorAffinity contains CPUs that cannot be bound with numactl --localalloc. {process.StandardError}", nameof(this.NumaProcessorAffinity)); + } + + this.LogSpecCpuStep(telemetryContext, "AffinityPreflightCompleted", ("exitCode", process.ExitCode)); + } + + string configurationFile = this.GetConfigurationFileName(); + string configurationPath = this.Combine(this.PackageDirectory, "config", configurationFile); + string configText = await this.fileSystem.File.ReadAllTextAsync(configurationPath); + const string rateSection = "intrate,fprate:"; + if (!configText.Contains(rateSection, StringComparison.OrdinalIgnoreCase)) + { + throw new WorkloadException($"SPEC CPU configuration '{configurationPath}' is missing the '{rateSection}' section required for processor affinity."); + } + + string bind = useCopyNumber ? string.Empty : $" bind = {string.Join(",", cores)}\n"; + string submit = $" submit = numactl --localalloc --physcpubind={copyCpu} -- $command"; + configText = configText.Replace(rateSection, $"{rateSection}\n{bind}{submit}", StringComparison.OrdinalIgnoreCase); + + string configId = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join(",", cores)))).Substring(0, 16); + string pinnedConfigFile = $"{Path.GetFileNameWithoutExtension(configurationFile)}-affinity-{configId}.cfg"; + await this.fileSystem.File.WriteAllTextAsync(this.Combine(this.PackageDirectory, "config", pinnedConfigFile), configText, cancellationToken); + this.LogSpecCpuStep(telemetryContext, "ConfigGenerated", ("configurationFile", pinnedConfigFile), ("cpuIds", string.Join(",", cores)), ("submit", submit)); + return pinnedConfigFile; + } + + private static bool TryParseNumaTopology(string topology, IReadOnlyList selectedCpus, out IReadOnlyList parsedNodes) + { + parsedNodes = null; + Match header = Regex.Match(topology, @"(?m)^available:[ \t]*(?\d+)[ \t]+nodes\b"); + if (!header.Success || !int.TryParse(header.Groups["count"].Value, out int nodeCount) || nodeCount == 0) + { + return false; + } + + Dictionary nodes = new Dictionary(); + MatchCollection entries = Regex.Matches(topology, @"(?m)^node[ \t]+(?\d+)[ \t]+(?cpus|size|free):[ \t]*(?[^\r\n]*)"); + foreach (Match entry in entries) + { + if (!int.TryParse(entry.Groups["id"].Value, out int nodeId)) + { + return false; + } + + if (!nodes.TryGetValue(nodeId, out NumaNode node)) + { + node = new NumaNode(nodeId); + nodes.Add(nodeId, node); + } + + string field = entry.Groups["field"].Value; + string value = entry.Groups["value"].Value.Trim(); + if (field == "cpus") + { + if (node.Cpus != null) + { + return false; + } + + List cpus = new List(); + foreach (string cpu in value.Split((char[])null, StringSplitOptions.RemoveEmptyEntries)) + { + if (!int.TryParse(cpu, out int cpuId) || cpuId < 0) + { + return false; + } + + cpus.Add(cpuId); + } + + node.Cpus = cpus; + } + else + { + Match memory = Regex.Match(value, @"^(?\d+(?:\.\d+)?)[ \t]+MB$"); + if (!memory.Success || !double.TryParse(memory.Groups["mb"].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out double megabytes)) + { + return false; + } + + if (field == "size") + { + if (node.SizeMb.HasValue) + { + return false; + } + + node.SizeMb = megabytes; + } + else + { + if (node.FreeMb.HasValue) + { + return false; + } + + node.FreeMb = megabytes; + } + } + } + + if (nodes.Count != nodeCount || nodes.Values.Any(node => node.Cpus == null || !node.SizeMb.HasValue || !node.FreeMb.HasValue)) + { + return false; + } + + if (nodes.Values.SelectMany(node => node.Cpus).Distinct().Count() != nodes.Values.Sum(node => node.Cpus.Count) + || selectedCpus.Any(cpu => nodes.Values.Count(node => node.Cpus.Contains(cpu)) != 1)) + { + return false; + } + + parsedNodes = nodes.Values.OrderBy(node => node.Id).ToList(); + return true; + } + + private void LogSpecCpuStep(EventContext telemetryContext, string step, params (string Name, object Value)[] details) + { + EventContext stepContext = telemetryContext.Clone() + .AddContext("scenario", this.Scenario ?? string.Empty) + .AddContext("specProfile", this.SpecProfile ?? string.Empty) + .AddContext("benchmarks", this.Benchmarks ?? string.Empty) + .AddContext("copies", this.CopyCount) + .AddContext("pinningEnabled", !string.IsNullOrWhiteSpace(this.NumaProcessorAffinity)); + + foreach ((string name, object value) in details) + { + stepContext.AddContext(name, value); + } + + this.Logger.LogMessage($"{this.TypeName}.{step}", LogLevel.Information, stepContext); + } + + private void LogSpecCpuFailure(EventContext telemetryContext, string stage, Exception exception) + { + EventContext errorContext = telemetryContext.Clone() + .AddContext("scenario", this.Scenario ?? string.Empty) + .AddContext("specProfile", this.SpecProfile ?? string.Empty) + .AddContext("stage", stage) + .AddError(exception); + + this.Logger.LogMessage($"{this.TypeName}.Failed", LogLevel.Error, errorContext); + } + private async Task WriteSpecCpuConfigAsync(CancellationToken cancellationToken) { string configurationFile = this.GetConfigurationFileName(); @@ -710,6 +1105,22 @@ public bool SpecCpuInitialized } } + private sealed class NumaNode + { + public NumaNode(int id) + { + this.Id = id; + } + + public int Id { get; } + + public List Cpus { get; set; } + + public double? SizeMb { get; set; } + + public double? FreeMb { get; set; } + } + private static class SpecCpuConfigPlaceholder { public const string BaseOptimizingFlags = "$BaseOptimizingFlags$"; diff --git a/src/VirtualClient/VirtualClient.Core.UnitTests/Logging/MetricsCsvFileLoggerTests.cs b/src/VirtualClient/VirtualClient.Core.UnitTests/Logging/MetricsCsvFileLoggerTests.cs index 63ab50d5c8..c85cdd7da7 100644 --- a/src/VirtualClient/VirtualClient.Core.UnitTests/Logging/MetricsCsvFileLoggerTests.cs +++ b/src/VirtualClient/VirtualClient.Core.UnitTests/Logging/MetricsCsvFileLoggerTests.cs @@ -58,6 +58,33 @@ public void MetricsCsvFileLoggerUsesTheExpectedColumnNamesForTheCsvFiles() CollectionAssert.AreEqual(expectedColumnHeaders, MetricsCsvFileLogger.CsvFields.Select(field => field.ColumnName)); } + [Test] + public void MetricsCsvFileLoggerDoesNotExportNumaCpuListsFromMetricMetadata() + { + EventContext context = new EventContext(Guid.NewGuid(), new Dictionary + { + ["toolName"] = "SPECcpu", + ["scenarioName"] = "ExecuteSPECBenchmark", + ["metricName"] = "node-cpus-0", + ["metricValue"] = 4, + ["metricUnit"] = "count", + ["metadata_metrics"] = new Dictionary + { + ["cpuIds"] = "0,1,48,49", + ["selectedCpuIds"] = "0,48", + ["processorAffinity"] = "0,1,48,49" + } + }); + + string csv = MetricsCsvFileLogger.CreateMessage(context); + + StringAssert.Contains("\"SPECcpu\"", csv); + StringAssert.Contains("\"node-cpus-0\",\"4\",\"count\"", csv); + StringAssert.DoesNotContain("0,1,48,49", csv); + StringAssert.DoesNotContain("0,48", csv); + StringAssert.DoesNotContain("cpuIds", csv); + } + [Test] public void MetricsCsvFileLoggerWritesTheExpectedCsvToFiles_1() { diff --git a/src/VirtualClient/VirtualClient.Core.UnitTests/ProcessExtensionsAffinityTests.cs b/src/VirtualClient/VirtualClient.Core.UnitTests/ProcessExtensionsAffinityTests.cs index 2d35af8ede..9510e9179d 100644 --- a/src/VirtualClient/VirtualClient.Core.UnitTests/ProcessExtensionsAffinityTests.cs +++ b/src/VirtualClient/VirtualClient.Core.UnitTests/ProcessExtensionsAffinityTests.cs @@ -57,6 +57,24 @@ public void CreateProcessWithAffinityCreatesExpectedBashCommandOnLinux() } } + [Test] + public void CreateProcessAffinityOverloadCreatesExpectedBashCommandOnLinux() + { + this.SetupDefaults(PlatformID.Unix); + + ProcessAffinityConfiguration config = ProcessAffinityConfiguration.Create(PlatformID.Unix, new[] { 0, 1, 2 }); + + using (IProcessProxy process = this.mockFixture.ProcessManager.CreateProcess( + "bash -c", + "myworkload --option1=value", + "/home/user/workdir", + config)) + { + Assert.AreEqual("bash -c \"numactl -C 0-2 myworkload --option1=value\"", process.StartInfo.FileName); + Assert.AreEqual("/home/user/workdir", process.StartInfo.WorkingDirectory); + } + } + [Test] public void CreateProcessWithAffinityThrowsOnNullConfiguration() { @@ -109,6 +127,26 @@ public void CreateElevatedProcessWithAffinityCreatesExpectedCommandOnLinux() } } + [Test] + public void CreateProcessElevatedAffinityOverloadCreatesExpectedCommandOnLinux() + { + this.SetupDefaults(PlatformID.Unix); + + ProcessAffinityConfiguration config = ProcessAffinityConfiguration.Create(PlatformID.Unix, new[] { 0, 1, 2 }); + + using (IProcessProxy process = this.mockFixture.ProcessManager.CreateProcess( + PlatformID.Unix, + "bash -c", + "myworkload --option1=value", + "/home/user/workdir", + config)) + { + Assert.AreEqual("sudo", process.StartInfo.FileName); + Assert.AreEqual("bash -c \"numactl -C 0-2 myworkload --option1=value\"", process.StartInfo.Arguments); + Assert.AreEqual("/home/user/workdir", process.StartInfo.WorkingDirectory); + } + } + [Test] public void CreateElevatedProcessWithAffinityHandlesComplexArguments() { diff --git a/src/VirtualClient/VirtualClient.Core/ProcessExtensions.cs b/src/VirtualClient/VirtualClient.Core/ProcessExtensions.cs index 01344e3da2..15df14d3f0 100644 --- a/src/VirtualClient/VirtualClient.Core/ProcessExtensions.cs +++ b/src/VirtualClient/VirtualClient.Core/ProcessExtensions.cs @@ -154,6 +154,36 @@ public static IProcessProxy CreateElevatedProcessWithAffinity(this ProcessManage } } + /// + /// Creates a Linux process with CPU affinity binding to specific cores. + /// + /// The process manager used to create the process. + /// The command to run. + /// The command line arguments to supply to the command. + /// The working directory for the command. + /// The CPU affinity configuration specifying which cores to bind to. + /// A process proxy with CPU affinity applied via numactl. + public static IProcessProxy CreateProcess(this ProcessManager processManager, string command, string arguments, string workingDir, ProcessAffinityConfiguration affinityConfig) + { + return processManager.CreateProcessWithAffinity(command, arguments, workingDir, affinityConfig); + } + + /// + /// Creates a Linux process with CPU affinity binding and elevated privileges when needed. + /// + /// The process manager used to create the process. + /// The OS platform. + /// The command to run. + /// The command line arguments to supply to the command. + /// The working directory for the command. + /// The CPU affinity configuration specifying which cores to bind to. + /// The username to use for running the command (Linux only). + /// A process proxy with CPU affinity and elevated privileges applied. + public static IProcessProxy CreateProcess(this ProcessManager processManager, PlatformID platform, string command, string arguments, string workingDir, ProcessAffinityConfiguration affinityConfig, string username = null) + { + return processManager.CreateElevatedProcessWithAffinity(platform, command, arguments, workingDir, affinityConfig, username); + } + /// /// Applies Windows CPU affinity to a running process. /// This should be called after the process has started. diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json index 4af18580bd..5d9e477040 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json @@ -10,7 +10,7 @@ "RecommendedMinimumExecutionTime": "(4-cores)=02:00:00,(16-cores)=05:00:00,(64-cores)=10:00:00", "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", "SupportedOperatingSystems": "AzureLinux,CentOS,Debian,OpenSuse,RedHat,Ubuntu,Windows", - "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24. SPEC CPU 2017 was not initially designed for win-arm64.", + "Notes": "Private package speccpu.2017.1.1.9-danielbowers.zip supports Ubuntu 24; SPEC CPU 2017 was not designed for win-arm64. NumaProcessorAffinity (Linux rate only): 'symmetric' balances copies across NUMA nodes, one CPU each; explicit CPU IDs pin copies manually; empty (default) does not pin.", "Limitations": "The following benchmarks don't work on Windows: 507, 511, 521, 526, 527, 538" }, "ParameterAliases": { @@ -25,6 +25,7 @@ "RunPeak": false, "ThreadCount": "{LogicalCoreCount}", "CopyCount": "{LogicalCoreCount}", + "NumaProcessorAffinity": "", "BaseOptimizingFlags": "-O2 -march={calculate((\"{Architecture}\" == \"x64\") ? \"x86-64-v3\" : \"armv8.2-a\")} -flto -frecord-gcc-switches", "PeakOptimizingFlags": "-O3 -march={calculate((\"{Architecture}\" == \"x64\") ? \"x86-64-v3\" : \"armv8.2-a\")} -flto -frecord-gcc-switches" }, @@ -40,6 +41,7 @@ "RunPeak": "$.Parameters.RunPeak", "ThreadCount": "$.Parameters.ThreadCount", "CopyCount": "$.Parameters.CopyCount", + "NumaProcessorAffinity": "$.Parameters.NumaProcessorAffinity", "BaseOptimizingFlags": "$.Parameters.BaseOptimizingFlags", "PeakOptimizingFlags": "$.Parameters.PeakOptimizingFlags" } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json index f531861c8f..c3dfdb6185 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json @@ -10,7 +10,7 @@ "RecommendedMinimumExecutionTime": "(8-cores)=04:00:00,(16-cores)=08:00:00,(64-cores)=10:00:00", "SupportedPlatforms": "linux-x64,linux-arm64,win-x64", "SupportedOperatingSystems": "AzureLinux,CentOS,Debian,OpenSuse,RedHat,Ubuntu,Windows", - "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24. SPEC CPU 2017 was not initially designed for win-arm64.", + "Notes": "Private package speccpu.2017.1.1.9-danielbowers.zip supports Ubuntu 24; SPEC CPU 2017 was not designed for win-arm64. NumaProcessorAffinity (Linux rate only): 'symmetric' balances copies across NUMA nodes, one CPU each; explicit CPU IDs pin copies manually; empty (default) does not pin.", "Limitations": "The following benchmarks don't work on Windows: 500, 502, 520, 523, 531" }, "ParameterAliases": { @@ -25,6 +25,7 @@ "RunPeak": false, "ThreadCount": "{LogicalCoreCount}", "CopyCount": "{LogicalCoreCount}", + "NumaProcessorAffinity": "", "BaseOptimizingFlags": "-O2 -march={calculate((\"{Architecture}\" == \"x64\") ? \"x86-64-v3\" : \"armv8.2-a\")} -flto -frecord-gcc-switches", "PeakOptimizingFlags": "-O3 -march={calculate((\"{Architecture}\" == \"x64\") ? \"x86-64-v3\" : \"armv8.2-a\")} -flto -frecord-gcc-switches" }, @@ -40,6 +41,7 @@ "RunPeak": "$.Parameters.RunPeak", "ThreadCount": "$.Parameters.ThreadCount", "CopyCount": "$.Parameters.CopyCount", + "NumaProcessorAffinity": "$.Parameters.NumaProcessorAffinity", "BaseOptimizingFlags": "$.Parameters.BaseOptimizingFlags", "PeakOptimizingFlags": "$.Parameters.PeakOptimizingFlags" } diff --git a/website/docs/workloads/speccpu/speccpu-profiles.md b/website/docs/workloads/speccpu/speccpu-profiles.md index c4e945e651..884e9b868f 100644 --- a/website/docs/workloads/speccpu/speccpu-profiles.md +++ b/website/docs/workloads/speccpu/speccpu-profiles.md @@ -44,6 +44,7 @@ for evaluating the performance of the CPU for processing calculations. | RunPeak | Optional. True to run the workload 'Peak' scenario, False to run the workload 'Base' scenario. | false (Base) | | ThreadCount | Optional. Determines the number of threads to use for running the benchmark. | # logical processors | | CopyCount | Optional. Determines the number of copies of the benchmark to run concurrently. | # logical processors | + | NumaProcessorAffinity | Optional. Linux SPECrate only. Set to `symmetric` to balance copies across NUMA nodes, one CPU per copy using `numactl`, or provide CPU IDs to pin copies manually. Empty leaves copies unpinned. | empty | | BaseOptimizationFlags | Optional. Optimization flags to pass to the GCC compiler when running the 'Base' scenario |
  • X64 systems: -g -O2 -march=x86-64-v3 -flto -frecord-gcc-switches
  • ARM64 systems: -g -O2 -march=armv8.2-a -flto -frecord-gcc-switches
| | PeakOptimizationFlags | Optional. Optimization flags to pass to the GCC compiler when running the 'Peak' scenario |
  • X64 systems: -g -O3 -march=x86-64-v3 -flto -frecord-gcc-switches
  • ARM64 systems: -g -O3 -march=armv8.2-a -flto -frecord-gcc-switches
| @@ -64,6 +65,7 @@ for evaluating the performance of the CPU for processing calculations. * **Notes** On Windows, only the following benchmarks are supported: 503.bwaves_r, 508.namd_r, 510.parest_r, 519.lbm_r, 544.nab_r, 549.fotonik3d_r, 554.roms_r. + The Linux `NumaProcessorAffinity` examples in the INTRATE section below also apply to this profile. ``` bash # Execute the workload profile @@ -155,6 +157,7 @@ for evaluating the performance of the CPU for processing calculations. | RunPeak | Optional. True to run the workload 'Peak' scenario, False to run the workload 'Base' scenario. | false (Base) | | ThreadCount | Optional. Determines the number of threads to use for running the benchmark. | # logical processors | | CopyCount | Optional. Determines the number of copies of the benchmark to run concurrently. | # logical processors | + | NumaProcessorAffinity | Optional. Linux SPECrate only. Set to `symmetric` to balance copies across NUMA nodes, one CPU per copy using `numactl`, or provide CPU IDs to pin copies manually. Empty leaves copies unpinned. | empty | | BaseOptimizationFlags | Optional. Optimization flags to pass to the GCC compiler when running the 'Base' scenario |
  • X64 systems: -g -O2 -march=x86-64-v3 -flto -frecord-gcc-switches
  • ARM64 systems: -g -O2 -march=armv8.2-a -flto -frecord-gcc-switches
| | PeakOptimizationFlags | Optional. Optimization flags to pass to the GCC compiler when running the 'Peak' scenario |
  • X64 systems: -g -O3 -march=x86-64-v3 -flto -frecord-gcc-switches
  • ARM64 systems: -g -O3 -march=armv8.2-a -flto -frecord-gcc-switches
| @@ -174,10 +177,17 @@ for evaluating the performance of the CPU for processing calculations. * **Notes** On Windows, only the following benchmarks are supported: 505.mcf_r, 525.x264_r, 541.leela_r, 548.exchange2_r, 557.xz_r. + On Linux, `NumaProcessorAffinity=symmetric` selects CPUs from `numactl --hardware`, giving each node as even a share of `CopyCount` as its available CPUs allow. If `CopyCount` is not divisible by the number of nodes, lower-numbered nodes receive the extra copies. Pinned copies use `numactl --localalloc` for local memory allocation. These examples assume 96 CPUs on two NUMA nodes: CPUs 0-47 on node 0 and 48-95 on node 1. Check the actual topology with `numactl --hardware`. ``` bash - # Execute the workload profile - ./VirtualClient --profile=PERF-SPECCPU-INTRATE.json --iterations=1 --logger=csv + # Two INTRATE copies: CPU 0 on node 0 and CPU 48 on node 1. + ./VirtualClient --profile=PERF-SPECCPU-INTRATE.json --iterations=1 --parameters="CopyCount=2,,,NumaProcessorAffinity=symmetric" --logger=csv + + # Four INTRATE copies: CPUs 0-1 on node 0 and 48-49 on node 1. + ./VirtualClient --profile=PERF-SPECCPU-INTRATE.json --iterations=1 --parameters="CopyCount=4,,,NumaProcessorAffinity=symmetric" --logger=csv + + # Four FPRATE copies: the same two-per-node distribution. + ./VirtualClient --profile=PERF-SPECCPU-FPRATE.json --iterations=1 --parameters="CopyCount=4,,,NumaProcessorAffinity=symmetric" --logger=csv ``` ## PERF-SPECCPU-INTSPEED.json