Skip to content

Commit b48a9ca

Browse files
authored
Merge pull request #1046 from ahoppen/ahoppen/skip-unless
Add infrastructure for skipping tests if the host toolchain doesn't support them
2 parents fb87812 + 0c79a6b commit b48a9ca

17 files changed

+333
-178
lines changed

Package.swift

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,9 @@ let package = Package(
237237
name: "SKTestSupport",
238238
dependencies: [
239239
"CSKTestSupport",
240+
"LanguageServerProtocol",
240241
"LSPTestSupport",
242+
"LSPLogging",
241243
"SKCore",
242244
"SourceKitLSP",
243245
.product(name: "ISDBTestSupport", package: "indexstore-db"),

Sources/SKTestSupport/LongTestsEnabled.swift

Lines changed: 0 additions & 20 deletions
This file was deleted.
Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// This source file is part of the Swift.org open source project
4+
//
5+
// Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors
6+
// Licensed under Apache License v2.0 with Runtime Library Exception
7+
//
8+
// See https://swift.org/LICENSE.txt for license information
9+
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
10+
//
11+
//===----------------------------------------------------------------------===//
12+
13+
import Foundation
14+
import LSPLogging
15+
import LanguageServerProtocol
16+
import RegexBuilder
17+
@_spi(Testing) import SKCore
18+
import XCTest
19+
20+
import enum PackageLoading.Platform
21+
import struct TSCBasic.AbsolutePath
22+
import class TSCBasic.Process
23+
import enum TSCBasic.ProcessEnv
24+
25+
// MARK: - Skip checks
26+
27+
/// Namespace for functions that are used to skip unsupported tests.
28+
public enum SkipUnless {
29+
private enum FeatureCheckResult {
30+
case featureSupported
31+
case featureUnsupported(skipMessage: String)
32+
}
33+
34+
/// For any feature that has already been evaluated, the result of whether or not it should be skipped.
35+
private static var checkCache: [String: FeatureCheckResult] = [:]
36+
37+
/// Throw an `XCTSkip` if any of the following conditions hold
38+
/// - The Swift version of the toolchain used for testing (`ToolchainRegistry.forTesting.default`) is older than
39+
/// `swiftVersion`
40+
/// - The Swift version of the toolchain used for testing is equal to `swiftVersion` and `featureCheck` returns
41+
/// `false`. This is used for features that are introduced in `swiftVersion` but are not present in all toolchain
42+
/// snapshots.
43+
///
44+
/// Having the version check indicates when the check tests can be removed (namely when the minimum required version
45+
/// to test sourcekit-lsp is above `swiftVersion`) and it ensures that tests can’t stay in the skipped state over
46+
/// multiple releases.
47+
///
48+
/// Independently of these checks, the tests are never skipped in Swift CI (identified by the presence of the `SWIFTCI_USE_LOCAL_DEPS` environment). Swift CI is assumed to always build its own toolchain, which is thus
49+
/// guaranteed to be up-to-date.
50+
private static func skipUnlessSupportedByToolchain(
51+
swiftVersion: SwiftVersion,
52+
featureName: String,
53+
file: StaticString,
54+
line: UInt,
55+
featureCheck: () async throws -> Bool
56+
) async throws {
57+
let checkResult: FeatureCheckResult
58+
if let cachedResult = checkCache[featureName] {
59+
checkResult = cachedResult
60+
} else if ProcessEnv.block["SWIFTCI_USE_LOCAL_DEPS"] != nil {
61+
// Never skip tests in CI. Toolchain should be up-to-date
62+
checkResult = .featureSupported
63+
} else {
64+
guard let swiftc = await ToolchainRegistry.forTesting.default?.swiftc else {
65+
throw SwiftVersionParsingError.failedToFindSwiftc
66+
}
67+
68+
let toolchainSwiftVersion = try await getSwiftVersion(swiftc)
69+
let requiredSwiftVersion = SwiftVersion(swiftVersion.major, swiftVersion.minor)
70+
if toolchainSwiftVersion < requiredSwiftVersion {
71+
checkResult = .featureUnsupported(
72+
skipMessage: """
73+
Skipping because toolchain has Swift version \(toolchainSwiftVersion) \
74+
but test requires at least \(requiredSwiftVersion)
75+
"""
76+
)
77+
} else if toolchainSwiftVersion == requiredSwiftVersion {
78+
logger.info("Checking if feature '\(featureName)' is supported")
79+
if try await !featureCheck() {
80+
checkResult = .featureUnsupported(skipMessage: "Skipping because toolchain doesn't contain \(featureName)")
81+
} else {
82+
checkResult = .featureSupported
83+
}
84+
logger.info("Done checking if feature '\(featureName)' is supported")
85+
} else {
86+
checkResult = .featureSupported
87+
}
88+
}
89+
checkCache[featureName] = checkResult
90+
91+
if case .featureUnsupported(let skipMessage) = checkResult {
92+
throw XCTSkip(skipMessage, file: file, line: line)
93+
}
94+
}
95+
96+
public static func sourcekitdHasSemanticTokensRequest(
97+
file: StaticString = #file,
98+
line: UInt = #line
99+
) async throws {
100+
try await skipUnlessSupportedByToolchain(
101+
swiftVersion: SwiftVersion(5, 11),
102+
featureName: "semantic token support in sourcekitd",
103+
file: file,
104+
line: line
105+
) {
106+
let testClient = try await TestSourceKitLSPClient()
107+
let uri = DocumentURI.for(.swift)
108+
testClient.openDocument("func test() {}", uri: uri)
109+
do {
110+
_ = try await testClient.send(DocumentSemanticTokensRequest(textDocument: TextDocumentIdentifier(uri)))
111+
} catch let error as ResponseError {
112+
return !error.message.contains("unknown request: source.request.semantic_tokens")
113+
}
114+
return true
115+
}
116+
}
117+
118+
public static func sourcekitdSupportsRename(
119+
file: StaticString = #file,
120+
line: UInt = #line
121+
) async throws {
122+
try await skipUnlessSupportedByToolchain(
123+
swiftVersion: SwiftVersion(5, 11),
124+
featureName: "rename support in sourcekitd",
125+
file: file,
126+
line: line
127+
) {
128+
let testClient = try await TestSourceKitLSPClient()
129+
let uri = DocumentURI.for(.swift)
130+
let positions = testClient.openDocument("void 1️⃣test() {}", uri: uri)
131+
do {
132+
_ = try await testClient.send(
133+
RenameRequest(textDocument: TextDocumentIdentifier(uri), position: positions["1️⃣"], newName: "test2")
134+
)
135+
} catch let error as ResponseError {
136+
return error.message != "Running sourcekit-lsp with a version of sourcekitd that does not support rename"
137+
}
138+
return true
139+
}
140+
}
141+
142+
/// SwiftPM moved the location where it stores Swift modules to a subdirectory in
143+
/// https://github.com/apple/swift-package-manager/pull/7103.
144+
///
145+
/// sourcekit-lsp uses the built-in SwiftPM to synthesize compiler arguments and cross-module tests fail if the host
146+
/// toolchain’s SwiftPM stores the Swift modules on the top level but we synthesize compiler arguments expecting the
147+
/// modules to be in a `Modules` subdirectory.
148+
public static func swiftpmStoresModulesInSubdirectory(
149+
file: StaticString = #file,
150+
line: UInt = #line
151+
) async throws {
152+
try await skipUnlessSupportedByToolchain(
153+
swiftVersion: SwiftVersion(5, 11),
154+
featureName: "SwiftPM stores modules in subdirectory",
155+
file: file,
156+
line: line
157+
) {
158+
let workspace = try await SwiftPMTestWorkspace(
159+
files: ["test.swift": ""],
160+
build: true
161+
)
162+
let modulesDirectory = workspace.scratchDirectory
163+
.appendingPathComponent(".build")
164+
.appendingPathComponent("debug")
165+
.appendingPathComponent("Modules")
166+
.appendingPathComponent("MyLibrary.swiftmodule")
167+
return FileManager.default.fileExists(atPath: modulesDirectory.path)
168+
}
169+
}
170+
171+
public static func longTestsEnabled() throws {
172+
if let value = ProcessInfo.processInfo.environment["SKIP_LONG_TESTS"], value == "1" || value == "YES" {
173+
throw XCTSkip("Long tests disabled using the `SKIP_LONG_TESTS` environment variable")
174+
}
175+
}
176+
177+
public static func platformIsDarwin(_ message: String) throws {
178+
try XCTSkipUnless(Platform.current == .darwin, message)
179+
}
180+
}
181+
182+
// MARK: - Parsing Swift compiler version
183+
184+
fileprivate extension String {
185+
init?(bytes: [UInt8], encoding: Encoding) {
186+
self = bytes.withUnsafeBytes { buffer in
187+
guard let baseAddress = buffer.baseAddress else {
188+
return ""
189+
}
190+
let data = Data(bytes: baseAddress, count: buffer.count)
191+
return String(data: data, encoding: encoding)!
192+
}
193+
}
194+
}
195+
196+
/// A Swift version consisting of the major and minor component.
197+
fileprivate struct SwiftVersion: Comparable, CustomStringConvertible {
198+
let major: Int
199+
let minor: Int
200+
201+
static func < (lhs: SwiftVersion, rhs: SwiftVersion) -> Bool {
202+
return (lhs.major, lhs.minor) < (rhs.major, rhs.minor)
203+
}
204+
205+
init(_ major: Int, _ minor: Int) {
206+
self.major = major
207+
self.minor = minor
208+
}
209+
210+
var description: String {
211+
return "\(major).\(minor)"
212+
}
213+
}
214+
215+
fileprivate enum SwiftVersionParsingError: Error, CustomStringConvertible {
216+
case failedToFindSwiftc
217+
case failedToParseOutput(output: String?)
218+
219+
var description: String {
220+
switch self {
221+
case .failedToFindSwiftc:
222+
return "Default toolchain does not contain a swiftc executable"
223+
case .failedToParseOutput(let output):
224+
return """
225+
Failed to parse Swift version. Output of swift --version:
226+
\(output ?? "<empty>")
227+
"""
228+
}
229+
}
230+
}
231+
232+
/// Return the major and minor version of Swift for a `swiftc` compiler at `swiftcPath`.
233+
private func getSwiftVersion(_ swiftcPath: AbsolutePath) async throws -> SwiftVersion {
234+
let process = Process(args: swiftcPath.pathString, "--version")
235+
try process.launch()
236+
let result = try await process.waitUntilExit()
237+
let output = String(bytes: try result.output.get(), encoding: .utf8)
238+
let regex = Regex {
239+
"Apple Swift version "
240+
Capture { OneOrMore(.digit) }
241+
"."
242+
Capture { OneOrMore(.digit) }
243+
}
244+
guard let match = output?.firstMatch(of: regex) else {
245+
throw SwiftVersionParsingError.failedToParseOutput(output: output)
246+
}
247+
guard let major = Int(match.1), let minor = Int(match.2) else {
248+
throw SwiftVersionParsingError.failedToParseOutput(output: output)
249+
}
250+
return SwiftVersion(major, minor)
251+
}

Tests/SKCoreTests/ToolchainRegistryTests.swift

Lines changed: 8 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,7 @@ final class ToolchainRegistryTests: XCTestCase {
6767
}
6868

6969
func testFindXcodeDefaultToolchain() async throws {
70-
#if !os(macOS)
71-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
72-
#endif
70+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
7371
let fs = InMemoryFileSystem()
7472
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")
7573
let toolchains = xcodeDeveloper.appending(components: "Toolchains")
@@ -96,9 +94,7 @@ final class ToolchainRegistryTests: XCTestCase {
9694
}
9795

9896
func testFindNonXcodeDefaultToolchains() async throws {
99-
#if !os(macOS)
100-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
101-
#endif
97+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
10298
let fs = InMemoryFileSystem()
10399
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")
104100
let toolchains = xcodeDeveloper.appending(components: "Toolchains")
@@ -128,9 +124,7 @@ final class ToolchainRegistryTests: XCTestCase {
128124
}
129125

130126
func testIgnoreToolchainsWithWrongExtensions() async throws {
131-
#if !os(macOS)
132-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
133-
#endif
127+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
134128
let fs = InMemoryFileSystem()
135129
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")
136130
let toolchains = xcodeDeveloper.appending(components: "Toolchains")
@@ -159,9 +153,7 @@ final class ToolchainRegistryTests: XCTestCase {
159153

160154
}
161155
func testTwoToolchainsWithSameIdentifier() async throws {
162-
#if !os(macOS)
163-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
164-
#endif
156+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
165157

166158
let fs = InMemoryFileSystem()
167159
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")
@@ -192,9 +184,7 @@ final class ToolchainRegistryTests: XCTestCase {
192184
}
193185

194186
func testGloballyInstalledToolchains() async throws {
195-
#if !os(macOS)
196-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
197-
#endif
187+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
198188
let fs = InMemoryFileSystem()
199189

200190
try makeXCToolchain(
@@ -220,9 +210,7 @@ final class ToolchainRegistryTests: XCTestCase {
220210
}
221211

222212
func testFindToolchainBasedOnInstallPath() async throws {
223-
#if !os(macOS)
224-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
225-
#endif
213+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
226214
let fs = InMemoryFileSystem()
227215
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")
228216
let toolchains = xcodeDeveloper.appending(components: "Toolchains")
@@ -247,9 +235,7 @@ final class ToolchainRegistryTests: XCTestCase {
247235
}
248236

249237
func testDarwinToolchainOverride() async throws {
250-
#if !os(macOS)
251-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
252-
#endif
238+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
253239

254240
let fs = InMemoryFileSystem()
255241
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")
@@ -287,9 +273,7 @@ final class ToolchainRegistryTests: XCTestCase {
287273
}
288274

289275
func testCreateToolchainFromBinPath() async throws {
290-
#if !os(macOS)
291-
try XCTSkipIf(true, "Finding toolchains in Xcode is only supported on macOS")
292-
#endif
276+
try SkipUnless.platformIsDarwin("Finding toolchains in Xcode is only supported on macOS")
293277

294278
let fs = InMemoryFileSystem()
295279
let xcodeDeveloper = try AbsolutePath(validating: "/Applications/Xcode.app/Developer")

0 commit comments

Comments
 (0)