Skip to content

Commit

Permalink
Fix build issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@fortmarek
fortmarek committed Dec 16, 2024
1 parent a8beb04 commit 710a28d
Show file tree
Hide file tree
Showing 6 changed files with 158 additions and 111 deletions.
31 changes: 31 additions & 0 deletions .github/workflows/checks.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
name: Checks

on:
push:
branches:
- main
pull_request: {}

concurrency:
group: checks-${{ github.head_ref }}
cancel-in-progress: true

jobs:
test:
name: "Test on macOS"
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- name: Test
run: swift test

test_linux:
name: "Test on Linux"
runs-on: ubuntu-latest
container: swift:6.0.0
steps:
- uses: actions/checkout@v4
- name: Install SQLite
run: apt-get install -y libsqlite3-dev sqlite3
- name: Test
run: swift test
27 changes: 14 additions & 13 deletions Sources/PackageCollectionGenerator/PackageCollectionGenerate.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import TSCUtility
import Utilities

@main
public struct PackageCollectionGenerate: ParsableCommand {
public struct PackageCollectionGenerate: AsyncParsableCommand {
public static let configuration = CommandConfiguration(
abstract: "Generate a package collection from the given list of packages."
)
Expand Down Expand Up @@ -65,7 +65,7 @@ public struct PackageCollectionGenerate: ParsableCommand {

public init() {}

public func run() throws {
public func run() async throws {
Backtrace.install()

// Parse auth tokens
Expand All @@ -91,20 +91,21 @@ public struct PackageCollectionGenerate: ParsableCommand {
let githubPackageMetadataProvider = GitHubPackageMetadataProvider(authTokens: authTokens)

// Generate metadata for each package
let packages: [Model.Collection.Package] = input.packages.compactMap { package in
var packages: [Model.Collection.Package] = []
for package in input.packages {
do {
let packageMetadata = try self.generateMetadata(for: package, metadataProvider: githubPackageMetadataProvider, jsonDecoder: jsonDecoder)
let packageMetadata = try await self.generateMetadata(for: package, metadataProvider: githubPackageMetadataProvider, jsonDecoder: jsonDecoder)
print("\(packageMetadata)", verbose: self.verbose)

guard !packageMetadata.versions.isEmpty else {
printError("Skipping package \(package.url) because it does not have any valid versions.")
return nil
continue
}

return packageMetadata
packages.append(packageMetadata)
} catch {
printError("Failed to generate metadata for package \(package.url): \(error)")
return nil
continue
}
}

Expand Down Expand Up @@ -147,7 +148,7 @@ public struct PackageCollectionGenerate: ParsableCommand {

private func generateMetadata(for package: PackageCollectionGeneratorInput.Package,
metadataProvider: PackageMetadataProvider,
jsonDecoder: JSONDecoder) throws -> Model.Collection.Package {
jsonDecoder: JSONDecoder) async throws -> Model.Collection.Package {
print("Processing Package(\(package.url))", inColor: .cyan, verbose: self.verbose)

// Try to locate the directory where the repository might have been cloned to previously
Expand Down Expand Up @@ -178,7 +179,7 @@ public struct PackageCollectionGenerate: ParsableCommand {
try GitUtilities.clone(repositoryURL, to: gitDirectoryPath)
}

return try self.generateMetadata(
return try await self.generateMetadata(
for: package,
gitDirectoryPath: gitDirectoryPath,
metadataProvider: metadataProvider,
Expand All @@ -188,11 +189,11 @@ public struct PackageCollectionGenerate: ParsableCommand {
}

// Fallback to tmp directory if we cannot use the working directory for some reason or it's unspecified
return try withTemporaryDirectory(removeTreeOnDeinit: true) { tmpDir in
return try await withTemporaryDirectory(removeTreeOnDeinit: true) { tmpDir in
// Clone the package repository
try GitUtilities.clone(package.url.absoluteString, to: tmpDir)

return try self.generateMetadata(
return try await self.generateMetadata(
for: package,
gitDirectoryPath: tmpDir,
metadataProvider: metadataProvider,
Expand All @@ -204,10 +205,10 @@ public struct PackageCollectionGenerate: ParsableCommand {
private func generateMetadata(for package: PackageCollectionGeneratorInput.Package,
gitDirectoryPath: AbsolutePath,
metadataProvider: PackageMetadataProvider,
jsonDecoder: JSONDecoder) throws -> Model.Collection.Package {
jsonDecoder: JSONDecoder) async throws -> Model.Collection.Package {
var additionalMetadata: PackageBasicMetadata?
do {
additionalMetadata = try temp_await { callback in metadataProvider.get(package.url, callback: callback) }
additionalMetadata = try await metadataProvider.get(package.url)
} catch {
printError("Failed to fetch additional metadata: \(error)")
}
Expand Down
153 changes: 78 additions & 75 deletions ...s/PackageCollectionGenerator/PackageMetadataProviders/GitHubPackageMetadataProvider.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,87 +32,90 @@ struct GitHubPackageMetadataProvider: PackageMetadataProvider {
self.decoder = JSONDecoder.makeWithDefaults()
}

func get(_ packageURL: URL, callback: @escaping (Result<PackageBasicMetadata, Error>) -> Void) {
guard let baseURL = self.apiURL(packageURL.absoluteString) else {
return callback(.failure(Errors.invalidGitURL(packageURL)))
}

let metadataURL = baseURL
let readmeURL = baseURL.appendingPathComponent("readme")
let licenseURL = baseURL.appendingPathComponent("license")

let sync = DispatchGroup()
let results = ThreadSafeKeyValueStore<URL, Result<HTTPClientResponse, Error>>()

// get the main data
sync.enter()
var metadataHeaders = HTTPClientHeaders()
metadataHeaders.add(name: "Accept", value: "application/vnd.github.mercy-preview+json")
let metadataOptions = self.makeRequestOptions(validResponseCodes: [200, 401, 403, 404])
let hasAuthorization = metadataOptions.authorizationProvider?(metadataURL) != nil
self.httpClient.get(metadataURL, headers: metadataHeaders, options: metadataOptions) { result in
defer { sync.leave() }
results[metadataURL] = result
if case .success(let response) = result {
let apiLimit = response.headers.get("X-RateLimit-Limit").first.flatMap(Int.init) ?? -1
let apiRemaining = response.headers.get("X-RateLimit-Remaining").first.flatMap(Int.init) ?? -1
switch (response.statusCode, hasAuthorization, apiRemaining) {
case (_, _, 0):
results[metadataURL] = .failure(Errors.apiLimitsExceeded(metadataURL, apiLimit, apiRemaining))
case (401, true, _):
results[metadataURL] = .failure(Errors.invalidAuthToken(metadataURL))
case (401, false, _):
results[metadataURL] = .failure(Errors.permissionDenied(metadataURL))
case (403, _, _):
results[metadataURL] = .failure(Errors.permissionDenied(metadataURL))
case (404, _, _):
results[metadataURL] = .failure(Errors.notFound(metadataURL))
case (200, _, _):
// if successful, fan out multiple API calls
[readmeURL, licenseURL].forEach { url in
sync.enter()
var headers = HTTPClientHeaders()
headers.add(name: "Accept", value: "application/vnd.github.v3+json")
let options = self.makeRequestOptions(validResponseCodes: [200])
self.httpClient.get(url, headers: headers, options: options) { result in
defer { sync.leave() }
results[url] = result
func get(_ packageURL: URL) async throws -> PackageBasicMetadata {
try await withCheckedThrowingContinuation { continuation in

guard let baseURL = self.apiURL(packageURL.absoluteString) else {
return continuation.resume(throwing: Errors.invalidGitURL(packageURL))
}

let metadataURL = baseURL
let readmeURL = baseURL.appendingPathComponent("readme")
let licenseURL = baseURL.appendingPathComponent("license")

let sync = DispatchGroup()
let results = ThreadSafeKeyValueStore<URL, Result<HTTPClientResponse, Error>>()

// get the main data
sync.enter()
var metadataHeaders = HTTPClientHeaders()
metadataHeaders.add(name: "Accept", value: "application/vnd.github.mercy-preview+json")
let metadataOptions = self.makeRequestOptions(validResponseCodes: [200, 401, 403, 404])
let hasAuthorization = metadataOptions.authorizationProvider?(metadataURL) != nil
self.httpClient.get(metadataURL, headers: metadataHeaders, options: metadataOptions) { result in
defer { sync.leave() }
results[metadataURL] = result
if case .success(let response) = result {
let apiLimit = response.headers.get("X-RateLimit-Limit").first.flatMap(Int.init) ?? -1
let apiRemaining = response.headers.get("X-RateLimit-Remaining").first.flatMap(Int.init) ?? -1
switch (response.statusCode, hasAuthorization, apiRemaining) {
case (_, _, 0):
results[metadataURL] = .failure(Errors.apiLimitsExceeded(metadataURL, apiLimit, apiRemaining))
case (401, true, _):
results[metadataURL] = .failure(Errors.invalidAuthToken(metadataURL))
case (401, false, _):
results[metadataURL] = .failure(Errors.permissionDenied(metadataURL))
case (403, _, _):
results[metadataURL] = .failure(Errors.permissionDenied(metadataURL))
case (404, _, _):
results[metadataURL] = .failure(Errors.notFound(metadataURL))
case (200, _, _):
// if successful, fan out multiple API calls
[readmeURL, licenseURL].forEach { url in
sync.enter()
var headers = HTTPClientHeaders()
headers.add(name: "Accept", value: "application/vnd.github.v3+json")
let options = self.makeRequestOptions(validResponseCodes: [200])
self.httpClient.get(url, headers: headers, options: options) { result in
defer { sync.leave() }
results[url] = result
}
}
default:
results[metadataURL] = .failure(Errors.invalidResponse(metadataURL, "Invalid status code: \(response.statusCode)"))
}
default:
results[metadataURL] = .failure(Errors.invalidResponse(metadataURL, "Invalid status code: \(response.statusCode)"))
}
}
}

// process results
sync.notify(queue: self.httpClient.configuration.callbackQueue) {
do {
// check for main request error state
switch results[metadataURL] {
case .none:
throw Errors.invalidResponse(metadataURL, "Response missing")
case .some(.failure(let error)):
throw error
case .some(.success(let metadataResponse)):
guard let metadata = try metadataResponse.decodeBody(GetRepositoryResponse.self, using: self.decoder) else {
throw Errors.invalidResponse(metadataURL, "Empty body")

// process results
sync.notify(queue: self.httpClient.configuration.callbackQueue) {
do {
// check for main request error state
switch results[metadataURL] {
case .none:
throw Errors.invalidResponse(metadataURL, "Response missing")
case .some(.failure(let error)):
throw error
case .some(.success(let metadataResponse)):
guard let metadata = try metadataResponse.decodeBody(GetRepositoryResponse.self, using: self.decoder) else {
throw Errors.invalidResponse(metadataURL, "Empty body")
}

let readme = try results[readmeURL]?.success?.decodeBody(Readme.self, using: self.decoder)
let license = try results[licenseURL]?.success?.decodeBody(License.self, using: self.decoder)

let model = PackageBasicMetadata(
summary: metadata.description,
keywords: metadata.topics,
readmeURL: readme?.downloadURL,
license: license.flatMap { .init(name: $0.license.spdxID, url: $0.downloadURL) }
)

continuation.resume(returning: model)
}

let readme = try results[readmeURL]?.success?.decodeBody(Readme.self, using: self.decoder)
let license = try results[licenseURL]?.success?.decodeBody(License.self, using: self.decoder)

let model = PackageBasicMetadata(
summary: metadata.description,
keywords: metadata.topics,
readmeURL: readme?.downloadURL,
license: license.flatMap { .init(name: $0.license.spdxID, url: $0.downloadURL) }
)

callback(.success(model))
} catch {
continuation.resume(throwing: error)
}
} catch {
return callback(.failure(error))
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion Sources/PackageCollectionGenerator/PackageMetadataProviders/PackageMetadataProvider.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import Foundation
import PackageCollectionsModel

protocol PackageMetadataProvider {
func get(_ packageURL: URL, callback: @escaping (Result<PackageBasicMetadata, Error>) -> Void)
func get(_ packageURL: URL) async throws -> PackageBasicMetadata
}

enum AuthTokenType: Hashable, CustomStringConvertible {
Expand Down
Loading

0 comments on commit 710a28d

Please sign in to comment.