Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix build issues #1

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Fix build issues #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
67e3681
Fix build issues
fortmarek Dec 16, 2024
7c7ec63
Run swiftformat
fortmarek Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions .github/workflows/checks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
name: Checks

on:
push:
branches:
- main
pull_request: {}

concurrency:
group: checks-${{ github.head_ref }}
cancel-in-progress: true

jobs:
test:
name: "Test on macOS"
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- name: Test
run: swift test

test_linux:
name: "Test on Linux"
runs-on: ubuntu-latest
container: swift:6.0.0
steps:
- uses: actions/checkout@v4
- name: Install SQLite
run: |
apt-get update
apt-get install -y libsqlite3-dev sqlite3
- name: Test
run: swift test
37 changes: 18 additions & 19 deletions Sources/PackageCollectionGenerator/PackageCollectionGenerate.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import TSCUtility
import Utilities

@main
public struct PackageCollectionGenerate: ParsableCommand {
public struct PackageCollectionGenerate: AsyncParsableCommand {
public static let configuration = CommandConfiguration(
abstract: "Generate a package collection from the given list of packages."
)
Expand Down Expand Up @@ -65,7 +65,7 @@ public struct PackageCollectionGenerate: ParsableCommand {

public init() {}

public func run() throws {
public func run() async throws {
Backtrace.install()

// Parse auth tokens
Expand All @@ -91,20 +91,21 @@ public struct PackageCollectionGenerate: ParsableCommand {
let githubPackageMetadataProvider = GitHubPackageMetadataProvider(authTokens: authTokens)

// Generate metadata for each package
let packages: [Model.Collection.Package] = input.packages.compactMap { package in
var packages: [Model.Collection.Package] = []
for package in input.packages {
do {
let packageMetadata = try self.generateMetadata(for: package, metadataProvider: githubPackageMetadataProvider, jsonDecoder: jsonDecoder)
let packageMetadata = try await self.generateMetadata(for: package, metadataProvider: githubPackageMetadataProvider, jsonDecoder: jsonDecoder)
print("\(packageMetadata)", verbose: self.verbose)

guard !packageMetadata.versions.isEmpty else {
printError("Skipping package \(package.url) because it does not have any valid versions.")
return nil
continue
}

return packageMetadata
packages.append(packageMetadata)
} catch {
printError("Failed to generate metadata for package \(package.url): \(error)")
return nil
continue
}
}

Expand Down Expand Up @@ -132,7 +133,7 @@ public struct PackageCollectionGenerate: ParsableCommand {
} catch {
outputAbsolutePath = try AbsolutePath(
validating: self.outputPath,
relativeTo: try AbsolutePath(validating: FileManager.default.currentDirectoryPath)
relativeTo: AbsolutePath(validating: FileManager.default.currentDirectoryPath)
)
}
let outputDirectory = outputAbsolutePath.parentDirectory
Expand All @@ -147,7 +148,7 @@ public struct PackageCollectionGenerate: ParsableCommand {

private func generateMetadata(for package: PackageCollectionGeneratorInput.Package,
metadataProvider: PackageMetadataProvider,
jsonDecoder: JSONDecoder) throws -> Model.Collection.Package {
jsonDecoder: JSONDecoder) async throws -> Model.Collection.Package {
print("Processing Package(\(package.url))", inColor: .cyan, verbose: self.verbose)

// Try to locate the directory where the repository might have been cloned to previously
Expand All @@ -158,7 +159,7 @@ public struct PackageCollectionGenerate: ParsableCommand {
} catch {
workingDirectoryAbsolutePath = try AbsolutePath(
validating: workingDirectoryPath,
relativeTo: try AbsolutePath(validating: FileManager.default.currentDirectoryPath)
relativeTo: AbsolutePath(validating: FileManager.default.currentDirectoryPath)
)
}

Expand All @@ -178,7 +179,7 @@ public struct PackageCollectionGenerate: ParsableCommand {
try GitUtilities.clone(repositoryURL, to: gitDirectoryPath)
}

return try self.generateMetadata(
return try await self.generateMetadata(
for: package,
gitDirectoryPath: gitDirectoryPath,
metadataProvider: metadataProvider,
Expand All @@ -188,11 +189,11 @@ public struct PackageCollectionGenerate: ParsableCommand {
}

// Fallback to tmp directory if we cannot use the working directory for some reason or it's unspecified
return try withTemporaryDirectory(removeTreeOnDeinit: true) { tmpDir in
return try await withTemporaryDirectory(removeTreeOnDeinit: true) { tmpDir in
// Clone the package repository
try GitUtilities.clone(package.url.absoluteString, to: tmpDir)

return try self.generateMetadata(
return try await self.generateMetadata(
for: package,
gitDirectoryPath: tmpDir,
metadataProvider: metadataProvider,
Expand All @@ -204,10 +205,10 @@ public struct PackageCollectionGenerate: ParsableCommand {
private func generateMetadata(for package: PackageCollectionGeneratorInput.Package,
gitDirectoryPath: AbsolutePath,
metadataProvider: PackageMetadataProvider,
jsonDecoder: JSONDecoder) throws -> Model.Collection.Package {
jsonDecoder: JSONDecoder) async throws -> Model.Collection.Package {
var additionalMetadata: PackageBasicMetadata?
do {
additionalMetadata = try temp_await { callback in metadataProvider.get(package.url, callback: callback) }
additionalMetadata = try await metadataProvider.get(package.url)
} catch {
printError("Failed to fetch additional metadata: \(error)")
}
Expand Down Expand Up @@ -268,8 +269,7 @@ public struct PackageCollectionGenerate: ParsableCommand {
excludedTargets: Set<String>,
signer: PackageCollectionModel.V1.Signer?,
gitDirectoryPath: AbsolutePath,
jsonDecoder: JSONDecoder) throws -> Model.Collection.Package.Version
{
jsonDecoder: JSONDecoder) throws -> Model.Collection.Package.Version {
// Check out the git tag
print("Checking out version \(version)", inColor: .yellow, verbose: self.verbose)
try ShellUtilities.run(Git.tool, "-C", gitDirectoryPath.pathString, "checkout", version)
Expand Down Expand Up @@ -301,8 +301,7 @@ public struct PackageCollectionGenerate: ParsableCommand {
private func defaultManifest(excludedProducts: Set<String>,
excludedTargets: Set<String>,
gitDirectoryPath: AbsolutePath,
jsonDecoder: JSONDecoder) throws -> Model.Collection.Package.Version.Manifest
{
jsonDecoder: JSONDecoder) throws -> Model.Collection.Package.Version.Manifest {
// Run `swift package describe --type json` to generate JSON package description
let packageDescriptionJSON = try ShellUtilities.run(ShellUtilities.shell, "-c", "cd \(gitDirectoryPath) && swift package describe --type json")
let packageDescription = try jsonDecoder.decode(PackageDescription.self, from: packageDescriptionJSON.data(using: .utf8) ?? Data())
Expand Down
147 changes: 75 additions & 72 deletions ...s/PackageCollectionGenerator/PackageMetadataProviders/GitHubPackageMetadataProvider.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,92 +32,95 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
self.decoder = JSONDecoder.makeWithDefaults()
}

func get(_ packageURL: URL, callback: @escaping (Result<PackageBasicMetadata, Error>) -> Void) {
guard let baseURL = self.apiURL(packageURL.absoluteString) else {
return callback(.failure(Errors.invalidGitURL(packageURL)))
}
func get(_ packageURL: URL) async throws -> PackageBasicMetadata {
try await withCheckedThrowingContinuation { continuation in

guard let baseURL = self.apiURL(packageURL.absoluteString) else {
return continuation.resume(throwing: Errors.invalidGitURL(packageURL))
}

let metadataURL = baseURL
let readmeURL = baseURL.appendingPathComponent("readme")
let licenseURL = baseURL.appendingPathComponent("license")

let sync = DispatchGroup()
let results = ThreadSafeKeyValueStore<URL, Result<HTTPClientResponse, Error>>()

// get the main data
sync.enter()
var metadataHeaders = HTTPClientHeaders()
metadataHeaders.add(name: "Accept", value: "application/vnd.github.mercy-preview+json")
let metadataOptions = self.makeRequestOptions(validResponseCodes: [200, 401, 403, 404])
let hasAuthorization = metadataOptions.authorizationProvider?(metadataURL) != nil
self.httpClient.get(metadataURL, headers: metadataHeaders, options: metadataOptions) { result in
defer { sync.leave() }
results[metadataURL] = result
if case .success(let response) = result {
let apiLimit = response.headers.get("X-RateLimit-Limit").first.flatMap(Int.init) ?? -1
let apiRemaining = response.headers.get("X-RateLimit-Remaining").first.flatMap(Int.init) ?? -1
switch (response.statusCode, hasAuthorization, apiRemaining) {
case (_, _, 0):
results[metadataURL] = .failure(Errors.apiLimitsExceeded(metadataURL, apiLimit, apiRemaining))
case (401, true, _):
results[metadataURL] = .failure(Errors.invalidAuthToken(metadataURL))
case (401, false, _):
results[metadataURL] = .failure(Errors.permissionDenied(metadataURL))
case (403, _, _):
results[metadataURL] = .failure(Errors.permissionDenied(metadataURL))
case (404, _, _):
results[metadataURL] = .failure(Errors.notFound(metadataURL))
case (200, _, _):
// if successful, fan out multiple API calls
[readmeURL, licenseURL].forEach { url in
sync.enter()
var headers = HTTPClientHeaders()
headers.add(name: "Accept", value: "application/vnd.github.v3+json")
let options = self.makeRequestOptions(validResponseCodes: [200])
self.httpClient.get(url, headers: headers, options: options) { result in
defer { sync.leave() }
results[url] = result
let metadataURL = baseURL
let readmeURL = baseURL.appendingPathComponent("readme")
let licenseURL = baseURL.appendingPathComponent("license")

let sync = DispatchGroup()
let results = ThreadSafeKeyValueStore<URL, Result<HTTPClientResponse, Error>>()

// get the main data
sync.enter()
var metadataHeaders = HTTPClientHeaders()
metadataHeaders.add(name: "Accept", value: "application/vnd.github.mercy-preview+json")
let metadataOptions = self.makeRequestOptions(validResponseCodes: [200, 401, 403, 404])
let hasAuthorization = metadataOptions.authorizationProvider?(metadataURL) != nil
self.httpClient.get(metadataURL, headers: metadataHeaders, options: metadataOptions) { result in
defer { sync.leave() }
results[metadataURL] = result
if case .success(let response) = result {
let apiLimit = response.headers.get("X-RateLimit-Limit").first.flatMap(Int.init) ?? -1
let apiRemaining = response.headers.get("X-RateLimit-Remaining").first.flatMap(Int.init) ?? -1
switch (response.statusCode, hasAuthorization, apiRemaining) {
case (_, _, 0):
results[metadataURL] = .failure(Errors.apiLimitsExceeded(metadataURL, apiLimit, apiRemaining))
case (401, true, _):
results[metadataURL] = .failure(Errors.invalidAuthToken(metadataURL))
case (401, false, _):
results[metadataURL] = .failure(Errors.permissionDenied(metadataURL))
case (403, _, _):
results[metadataURL] = .failure(Errors.permissionDenied(metadataURL))
case (404, _, _):
results[metadataURL] = .failure(Errors.notFound(metadataURL))
case (200, _, _):
// if successful, fan out multiple API calls
for url in [readmeURL, licenseURL] {
sync.enter()
var headers = HTTPClientHeaders()
headers.add(name: "Accept", value: "application/vnd.github.v3+json")
let options = self.makeRequestOptions(validResponseCodes: [200])
self.httpClient.get(url, headers: headers, options: options) { result in
defer { sync.leave() }
results[url] = result
}
}
default:
results[metadataURL] = .failure(Errors.invalidResponse(metadataURL, "Invalid status code: \(response.statusCode)"))
}
default:
results[metadataURL] = .failure(Errors.invalidResponse(metadataURL, "Invalid status code: \(response.statusCode)"))
}
}
}

// process results
sync.notify(queue: self.httpClient.configuration.callbackQueue) {
do {
// check for main request error state
switch results[metadataURL] {
case .none:
throw Errors.invalidResponse(metadataURL, "Response missing")
case .some(.failure(let error)):
throw error
case .some(.success(let metadataResponse)):
guard let metadata = try metadataResponse.decodeBody(GetRepositoryResponse.self, using: self.decoder) else {
throw Errors.invalidResponse(metadataURL, "Empty body")
}
// process results
sync.notify(queue: self.httpClient.configuration.callbackQueue) {
do {
// check for main request error state
switch results[metadataURL] {
case .none:
throw Errors.invalidResponse(metadataURL, "Response missing")
case .some(.failure(let error)):
throw error
case .some(.success(let metadataResponse)):
guard let metadata = try metadataResponse.decodeBody(GetRepositoryResponse.self, using: self.decoder) else {
throw Errors.invalidResponse(metadataURL, "Empty body")
}

let readme = try results[readmeURL]?.success?.decodeBody(Readme.self, using: self.decoder)
let license = try results[licenseURL]?.success?.decodeBody(License.self, using: self.decoder)
let readme = try results[readmeURL]?.success?.decodeBody(Readme.self, using: self.decoder)
let license = try results[licenseURL]?.success?.decodeBody(License.self, using: self.decoder)

let model = PackageBasicMetadata(
summary: metadata.description,
keywords: metadata.topics,
readmeURL: readme?.downloadURL,
license: license.flatMap { .init(name: $0.license.spdxID, url: $0.downloadURL) }
)
let model = PackageBasicMetadata(
summary: metadata.description,
keywords: metadata.topics,
readmeURL: readme?.downloadURL,
license: license.flatMap { .init(name: $0.license.spdxID, url: $0.downloadURL) }
)

callback(.success(model))
continuation.resume(returning: model)
}
} catch {
continuation.resume(throwing: error)
}
} catch {
return callback(.failure(error))
}
}
}

internal func apiURL(_ url: String) -> Foundation.URL? {
func apiURL(_ url: String) -> Foundation.URL? {
if let gitURL = GitURL.from(url) {
return URL(string: "https://\(Self.apiHostPrefix)\(gitURL.host)/repos/\(gitURL.owner)/\(gitURL.repository)")
}
Expand Down
2 changes: 1 addition & 1 deletion Sources/PackageCollectionGenerator/PackageMetadataProviders/PackageMetadataProvider.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import Foundation
import PackageCollectionsModel

protocol PackageMetadataProvider {
func get(_ packageURL: URL, callback: @escaping (Result<PackageBasicMetadata, Error>) -> Void)
func get(_ packageURL: URL) async throws -> PackageBasicMetadata
}

enum AuthTokenType: Hashable, CustomStringConvertible {
Expand Down
2 changes: 1 addition & 1 deletion Sources/PackageCollectionSigner/PackageCollectionSign.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public struct PackageCollectionSign: AsyncParsableCommand {
try await self._run(signer: nil)
}

internal func _run(signer: PackageCollectionSigner?) async throws {
func _run(signer: PackageCollectionSigner?) async throws {
Backtrace.install()

guard !self.certChainPaths.isEmpty else {
Expand Down
Loading
Loading