Skip to content

Commit 72f7355

Browse files
tamasvajktamasvajk
authored
Merge pull request #15692 from tamasvajk/buildless/no-dotnet-sdk
C#: Download latest dotnet SDK when missing
2 parents 047a8b4 + 20f795c commit 72f7355

File tree

5 files changed

+91
-47
lines changed

5 files changed

+91
-47
lines changed

‎csharp/autobuilder/Semmle.Autobuild.CSharp.Tests/BuildScripts.cs

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -558,6 +558,8 @@ public void TestVcVarsAllBatFiles()
558558
[Fact]
559559
public void TestLinuxBuildlessExtractionSuccess()
560560
{
561+
actions.RunProcess["dotnet --list-sdks"] = 0;
562+
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
561563
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
562564
actions.FileExists["csharp.log"] = true;
563565
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -567,12 +569,14 @@ public void TestLinuxBuildlessExtractionSuccess()
567569
actions.EnumerateDirectories[@"C:\Project"] = "";
568570

569571
var autobuilder = CreateAutoBuilder(false, buildless: "true");
570-
TestAutobuilderScript(autobuilder, 0, 1);
572+
TestAutobuilderScript(autobuilder, 0, 2);
571573
}
572574

573575
[Fact]
574576
public void TestLinuxBuildlessExtractionFailed()
575577
{
578+
actions.RunProcess["dotnet --list-sdks"] = 0;
579+
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
576580
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 10;
577581
actions.FileExists["csharp.log"] = true;
578582
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -582,12 +586,14 @@ public void TestLinuxBuildlessExtractionFailed()
582586
actions.EnumerateDirectories[@"C:\Project"] = "";
583587

584588
var autobuilder = CreateAutoBuilder(false, buildless: "true");
585-
TestAutobuilderScript(autobuilder, 10, 1);
589+
TestAutobuilderScript(autobuilder, 10, 2);
586590
}
587591

588592
[Fact]
589593
public void TestLinuxBuildlessExtractionSolution()
590594
{
595+
actions.RunProcess["dotnet --list-sdks"] = 0;
596+
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
591597
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
592598
actions.FileExists["csharp.log"] = true;
593599
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -597,7 +603,28 @@ public void TestLinuxBuildlessExtractionSolution()
597603
actions.EnumerateDirectories[@"C:\Project"] = "";
598604

599605
var autobuilder = CreateAutoBuilder(false, buildless: "true");
600-
TestAutobuilderScript(autobuilder, 0, 1);
606+
TestAutobuilderScript(autobuilder, 0, 2);
607+
}
608+
609+
[Fact]
610+
public void TestLinuxBuildlessExtractionNoDotnet()
611+
{
612+
actions.RunProcess["dotnet --list-sdks"] = 1;
613+
actions.RunProcessOut["dotnet --list-sdks"] = "";
614+
actions.RunProcess[@"chmod u+x scratch/.dotnet/dotnet-install.sh"] = 0;
615+
actions.RunProcess[@"scratch/.dotnet/dotnet-install.sh --channel release --version 8.0.101 --install-dir scratch/.dotnet"] = 0;
616+
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone --dotnet scratch/.dotnet"] = 0;
617+
actions.FileExists["csharp.log"] = true;
618+
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
619+
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SOURCE_ARCHIVE_DIR"] = "";
620+
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_SCRATCH_DIR"] = "scratch";
621+
actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.sln";
622+
actions.EnumerateDirectories[@"C:\Project"] = "";
623+
actions.DownloadFiles.Add(("https://dot.net/v1/dotnet-install.sh", "scratch/.dotnet/dotnet-install.sh"));
624+
actions.CreateDirectories.Add(@"scratch/.dotnet");
625+
626+
var autobuilder = CreateAutoBuilder(false, buildless: "true");
627+
TestAutobuilderScript(autobuilder, 0, 4);
601628
}
602629

603630
private void SkipVsWhere()
@@ -888,6 +915,8 @@ public void TestSkipNugetMsBuild()
888915
[Fact]
889916
public void TestSkipNugetBuildless()
890917
{
918+
actions.RunProcess["dotnet --list-sdks"] = 0;
919+
actions.RunProcessOut["dotnet --list-sdks"] = "any version";
891920
actions.RunProcess[@"C:\codeql\csharp/tools/linux64/Semmle.Extraction.CSharp.Standalone"] = 0;
892921
actions.FileExists["csharp.log"] = true;
893922
actions.GetEnvironmentVariable["CODEQL_EXTRACTOR_CSHARP_TRAP_DIR"] = "";
@@ -897,7 +926,7 @@ public void TestSkipNugetBuildless()
897926
actions.EnumerateDirectories[@"C:\Project"] = "";
898927

899928
var autobuilder = CreateAutoBuilder(false, buildless: "true");
900-
TestAutobuilderScript(autobuilder, 0, 1);
929+
TestAutobuilderScript(autobuilder, 0, 2);
901930
}
902931

903932

‎csharp/autobuilder/Semmle.Autobuild.CSharp/CSharpAutobuilder.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public override BuildScript GetBuildScript()
5050
attempt = new BuildCommandRule(DotNetRule.WithDotNet).Analyse(this, false) & CheckExtractorRun(true);
5151
break;
5252
case CSharpBuildStrategy.Buildless:
53-
attempt = DotNetRule.WithDotNet(this, (dotNetPath, env) =>
53+
attempt = DotNetRule.WithDotNet(this, ensureDotNetAvailable: true, (dotNetPath, env) =>
5454
{
5555
// No need to check that the extractor has been executed in buildless mode
5656
return new StandaloneBuildRule(dotNetPath).Analyse(this, false);

‎csharp/autobuilder/Semmle.Autobuild.CSharp/Constants.cs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
namespace Semmle.Autobuild.CSharp
2+
{
3+
internal static class Constants
4+
{
5+
// The version number should be kept in sync with the version .NET version used for building the application.
6+
public const string LatestDotNetSdkVersion = "8.0.101";
7+
}
8+
}

‎csharp/autobuilder/Semmle.Autobuild.CSharp/DotNetRule.cs

Lines changed: 37 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool au
4646
builder.Log(Severity.Info, "Attempting to build using .NET Core");
4747
}
4848

49-
return WithDotNet(builder, (dotNetPath, environment) =>
49+
return WithDotNet(builder, ensureDotNetAvailable: false, (dotNetPath, environment) =>
5050
{
5151
var ret = GetInfoCommand(builder.Actions, dotNetPath, environment);
5252
foreach (var projectOrSolution in builder.ProjectsOrSolutionsToBuild)
@@ -79,29 +79,29 @@ public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool au
7979
/// variables needed by the installed .NET Core (<code>null</code> when no variables
8080
/// are needed).
8181
/// </summary>
82-
public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<string?, IDictionary<string, string>?, BuildScript> f)
82+
public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, bool ensureDotNetAvailable, Func<string?, IDictionary<string, string>?, BuildScript> f)
8383
{
8484
var installDir = builder.Actions.PathCombine(FileUtils.GetTemporaryWorkingDirectory(builder.Actions.GetEnvironmentVariable, builder.Options.Language.UpperCaseName, out var _), ".dotnet");
85-
var installScript = DownloadDotNet(builder, installDir);
85+
var installScript = DownloadDotNet(builder, installDir, ensureDotNetAvailable);
8686
return BuildScript.Bind(installScript, installed =>
8787
{
88-
Dictionary<string, string>? env;
88+
var env = new Dictionary<string, string>
89+
{
90+
{ "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" },
91+
{ "MSBUILDDISABLENODEREUSE", "1" }
92+
};
8993
if (installed == 0)
9094
{
9195
// The installation succeeded, so use the newly installed .NET Core
9296
var path = builder.Actions.GetEnvironmentVariable("PATH");
9397
var delim = builder.Actions.IsWindows() ? ";" : ":";
94-
env = new Dictionary<string, string>{
95-
{ "DOTNET_MULTILEVEL_LOOKUP", "false" }, // prevent look up of other .NET Core SDKs
96-
{ "DOTNET_SKIP_FIRST_TIME_EXPERIENCE", "true" },
97-
{ "MSBUILDDISABLENODEREUSE", "1" },
98-
{ "PATH", installDir + delim + path }
99-
};
98+
env.Add("DOTNET_MULTILEVEL_LOOKUP", "false"); // prevent look up of other .NET Core SDKs
99+
env.Add("PATH", installDir + delim + path);
100100
}
101101
else
102102
{
103+
// The .NET SDK was not installed, either because the installation failed or because it was already installed.
103104
installDir = null;
104-
env = null;
105105
}
106106

107107
return f(installDir, env);
@@ -117,14 +117,14 @@ public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builde
117117
/// are needed).
118118
/// </summary>
119119
public static BuildScript WithDotNet(IAutobuilder<AutobuildOptionsShared> builder, Func<IDictionary<string, string>?, BuildScript> f)
120-
=> WithDotNet(builder, (_1, env) => f(env));
120+
=> WithDotNet(builder, ensureDotNetAvailable: false, (_, env) => f(env));
121121

122122
/// <summary>
123123
/// Returns a script for downloading relevant versions of the
124124
/// .NET Core SDK. The SDK(s) will be installed at <code>installDir</code>
125125
/// (provided that the script succeeds).
126126
/// </summary>
127-
private static BuildScript DownloadDotNet(IAutobuilder<AutobuildOptionsShared> builder, string installDir)
127+
private static BuildScript DownloadDotNet(IAutobuilder<AutobuildOptionsShared> builder, string installDir, bool ensureDotNetAvailable)
128128
{
129129
if (!string.IsNullOrEmpty(builder.Options.DotNetVersion))
130130
// Specific version supplied in configuration: always use that
@@ -152,7 +152,17 @@ private static BuildScript DownloadDotNet(IAutobuilder<AutobuildOptionsShared> b
152152
validGlobalJson = true;
153153
}
154154

155-
return validGlobalJson ? installScript : BuildScript.Failure;
155+
if (validGlobalJson)
156+
{
157+
return installScript;
158+
}
159+
160+
if (ensureDotNetAvailable)
161+
{
162+
return DownloadDotNetVersion(builder, installDir, Constants.LatestDotNetSdkVersion, needExactVersion: false);
163+
}
164+
165+
return BuildScript.Failure;
156166
}
157167

158168
/// <summary>
@@ -161,14 +171,25 @@ private static BuildScript DownloadDotNet(IAutobuilder<AutobuildOptionsShared> b
161171
///
162172
/// See https://docs.microsoft.com/en-us/dotnet/core/tools/dotnet-install-script.
163173
/// </summary>
164-
private static BuildScript DownloadDotNetVersion(IAutobuilder<AutobuildOptionsShared> builder, string path, string version)
174+
private static BuildScript DownloadDotNetVersion(IAutobuilder<AutobuildOptionsShared> builder, string path, string version, bool needExactVersion = true)
165175
{
166176
return BuildScript.Bind(GetInstalledSdksScript(builder.Actions), (sdks, sdksRet) =>
167177
{
168-
if (sdksRet == 0 && sdks.Count == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal))
178+
if (needExactVersion && sdksRet == 0 && sdks.Count == 1 && sdks[0].StartsWith(version + " ", StringComparison.Ordinal))
179+
{
169180
// The requested SDK is already installed (and no other SDKs are installed), so
170181
// no need to reinstall
171182
return BuildScript.Failure;
183+
}
184+
else if (!needExactVersion && sdksRet == 0 && sdks.Count > 0)
185+
{
186+
// there's at least one SDK installed, so no need to reinstall
187+
return BuildScript.Failure;
188+
}
189+
else if (!needExactVersion && sdksRet != 0)
190+
{
191+
builder.Log(Severity.Info, "No .NET Core SDK found.");
192+
}
172193

173194
builder.Log(Severity.Info, "Attempting to download .NET Core {0}", version);
174195

‎csharp/autobuilder/Semmle.Autobuild.CSharp/StandaloneBuildRule.cs

Lines changed: 12 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
using System.Linq;
2-
using Semmle.Autobuild.Shared;
1+
using Semmle.Autobuild.Shared;
32

43
namespace Semmle.Autobuild.CSharp
54
{
@@ -17,36 +16,23 @@ internal StandaloneBuildRule(string? dotNetPath)
1716

1817
public BuildScript Analyse(IAutobuilder<CSharpAutobuildOptions> builder, bool auto)
1918
{
20-
BuildScript GetCommand()
19+
if (builder.CodeQLExtractorLangRoot is null
20+
|| builder.CodeQlPlatform is null)
2121
{
22-
string standalone;
23-
if (builder.CodeQLExtractorLangRoot is not null && builder.CodeQlPlatform is not null)
24-
{
25-
standalone = builder.Actions.PathCombine(builder.CodeQLExtractorLangRoot, "tools", builder.CodeQlPlatform, "Semmle.Extraction.CSharp.Standalone");
26-
}
27-
else
28-
{
29-
return BuildScript.Failure;
30-
}
31-
32-
var cmd = new CommandBuilder(builder.Actions);
33-
cmd.RunCommand(standalone);
34-
35-
if (!string.IsNullOrEmpty(this.dotNetPath))
36-
{
37-
cmd.Argument("--dotnet");
38-
cmd.QuoteArgument(this.dotNetPath);
39-
}
40-
41-
return cmd.Script;
22+
return BuildScript.Failure;
4223
}
4324

44-
if (!builder.Options.Buildless)
25+
var standalone = builder.Actions.PathCombine(builder.CodeQLExtractorLangRoot, "tools", builder.CodeQlPlatform, "Semmle.Extraction.CSharp.Standalone");
26+
var cmd = new CommandBuilder(builder.Actions);
27+
cmd.RunCommand(standalone);
28+
29+
if (!string.IsNullOrEmpty(this.dotNetPath))
4530
{
46-
return BuildScript.Failure;
31+
cmd.Argument("--dotnet");
32+
cmd.QuoteArgument(this.dotNetPath);
4733
}
4834

49-
return GetCommand();
35+
return cmd.Script;
5036
}
5137
}
5238
}

0 commit comments

Comments
 (0)