From f7083a3c74a4aa1f6a0f4ab65265eb2f422a2cf0 Mon Sep 17 00:00:00 2001 From: Brad Slayter Date: Tue, 17 Sep 2024 08:28:07 -0500 Subject: [PATCH] Make remote config accessible to background agents (#947) Please review the release process for BrowserServicesKit [here](https://app.asana.com/0/1200194497630846/1200837094583426). **Required**: Task/Issue URL: https://app.asana.com/0/1203581873609357/1207165680693234/f iOS PR: https://github.com/duckduckgo/iOS/pull/3255 macOS PR: https://github.com/duckduckgo/macos-browser/pull/3124 What kind of version bump will this require?: Major **Optional**: Tech Design URL: CC: **Description**: This PR modularizes the config to allow it to be used in background processes **Steps to test this PR**: 1. Run the VPN and DBP background agents 2. Look for `Configuration` logs that indicate the config being downloaded or refreshed 3. If the config is setup correctly you can watch for the pixel test as well. 1. **OS Testing**: * [ ] iOS 14 * [ ] iOS 15 * [ ] iOS 16 * [ ] macOS 10.15 * [ ] macOS 11 * [ ] macOS 12 --- ###### Internal references: [Software Engineering Expectations](https://app.asana.com/0/59792373528535/199064865822552) [Technical Design Template](https://app.asana.com/0/59792373528535/184709971311943) --- .../BrowserServicesKit-Package.xcscheme | 24 ++ .../Features/PrivacyFeature.swift | 6 + .../Configuration/ConfigurationFetching.swift | 2 +- .../Configuration/ConfigurationStoring.swift | 2 + .../DefaultConfigurationManager.swift | 164 +++++++++++++ Sources/TestUtils/MockKeyValueStore.swift | 4 + .../ConfigurationManagerTests.swift | 224 ++++++++++++++++++ .../ConfigurationTests/Mocks/MockStore.swift | 4 + .../Mocks/MockStoreWithStorage.swift | 91 +++++++ 9 files changed, 520 insertions(+), 1 deletion(-) create mode 100644 Sources/Configuration/DefaultConfigurationManager.swift create mode 100644 Tests/ConfigurationTests/ConfigurationManagerTests.swift create mode 100644 Tests/ConfigurationTests/Mocks/MockStoreWithStorage.swift diff --git a/.swiftpm/xcode/xcshareddata/xcschemes/BrowserServicesKit-Package.xcscheme b/.swiftpm/xcode/xcshareddata/xcschemes/BrowserServicesKit-Package.xcscheme index cc56f8f16..0cad09202 100644 --- a/.swiftpm/xcode/xcshareddata/xcschemes/BrowserServicesKit-Package.xcscheme +++ b/.swiftpm/xcode/xcshareddata/xcschemes/BrowserServicesKit-Package.xcscheme @@ -483,6 +483,20 @@ ReferencedContainer = "container:"> + + + + + + + + URL + } diff --git a/Sources/Configuration/DefaultConfigurationManager.swift b/Sources/Configuration/DefaultConfigurationManager.swift new file mode 100644 index 000000000..89eddea27 --- /dev/null +++ b/Sources/Configuration/DefaultConfigurationManager.swift @@ -0,0 +1,164 @@ +// +// DefaultConfigurationManager.swift +// +// Copyright © 2024 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import os.log +import Combine +import Common +import Persistence + +public extension Logger { + static var config: Logger = { Logger(subsystem: Bundle.main.bundleIdentifier ?? "DuckDuckGo", category: "Configuration") }() +} + +open class DefaultConfigurationManager: NSObject { + public enum Error: Swift.Error { + + case timeout + case bloomFilterSpecNotFound + case bloomFilterBinaryNotFound + case bloomFilterPersistenceFailed + case bloomFilterExclusionsNotFound + case bloomFilterExclusionsPersistenceFailed + + public func withUnderlyingError(_ underlyingError: Swift.Error) -> Swift.Error { + let nsError = self as NSError + return NSError(domain: nsError.domain, code: nsError.code, userInfo: [NSUnderlyingErrorKey: underlyingError]) + } + + } + + public enum Constants { + + public static let downloadTimeoutSeconds = 60.0 * 5 +#if DEBUG + public static let refreshPeriodSeconds = 60.0 * 2 // 2 minutes +#else + public static let refreshPeriodSeconds = 60.0 * 30 // 30 minutes +#endif + public static let retryDelaySeconds = 60.0 * 60 * 1 // 1 hour delay before checking again if something went wrong last time + public static let refreshCheckIntervalSeconds = 60.0 // check if we need a refresh every minute + + static let lastUpdateDefaultsKey = "configuration.lastUpdateTime" + } + + private var defaults: KeyValueStoring + + public var fetcher: ConfigurationFetching + public var store: ConfigurationStoring + + public init(fetcher: ConfigurationFetching, store: ConfigurationStoring, defaults: KeyValueStoring) { + self.fetcher = fetcher + self.store = store + self.defaults = defaults + super.init() + NSFileCoordinator.addFilePresenter(self) + } + + deinit { + NSFileCoordinator.removeFilePresenter(self) + } + + public static let queue: DispatchQueue = DispatchQueue(label: "Configuration Manager") + public static let filePresenterOperationQueue = OperationQueue() + + public var lastUpdateTime: Date { + get { + defaults.object(forKey: Constants.lastUpdateDefaultsKey) as? Date ?? .distantPast + } + set { + defaults.set(newValue, forKey: Constants.lastUpdateDefaultsKey) + } + } + + private var timerCancellable: AnyCancellable? + private var refreshTask: Task? { + willSet { + refreshTask?.cancel() + } + } + public var lastRefreshCheckTime: Date = Date() + + public func start() { + Logger.config.debug("Starting configuration refresh timer") + refreshTask = Task.periodic(interval: Constants.refreshCheckIntervalSeconds) { + Self.queue.async { [weak self] in + self?.lastRefreshCheckTime = Date() + self?.refreshIfNeeded() + } + } + Task { + await refreshNow() + } + } + + /// Implement this in the subclass. + /// Use this method to fetch neccessary configurations and store them. + open func refreshNow(isDebug: Bool = false) async { + fatalError("refreshNow Must be implemented by subclass") + } + + @discardableResult + private func refreshIfNeeded() -> Task? { + guard isReadyToRefresh else { + Logger.config.debug("Configuration refresh is not needed at this time") + return nil + } + return Task { + await refreshNow() + } + } + + open var isReadyToRefresh: Bool { Date().timeIntervalSince(lastUpdateTime) > Constants.refreshPeriodSeconds } + + public func forceRefresh(isDebug: Bool = false) { + Task { + await refreshNow(isDebug: isDebug) + } + } + + /// Will try to update the config again at the regularly scheduled interval + /// **Note:** You must call `start()` on your `ConfigurationManager` instance for this to take effect. It relies on the internal refresh loop of the + /// `DefaultConfigurationManager` class + public func tryAgainLater() { + lastUpdateTime = Date() + } + + /// Will try to update the config again after `Constants.retryDelaySeconds` + /// **Note:** You must call `start()` on your `ConfigurationManager` instance for this to take effect. It relies on the internal refresh loop of the + /// `DefaultConfigurationManager` class + public func tryAgainSoon() { + // Set the last update time to in the past so it triggers again sooner + lastUpdateTime = Date(timeIntervalSinceNow: Constants.refreshPeriodSeconds - Constants.retryDelaySeconds) + } +} + +extension DefaultConfigurationManager: NSFilePresenter { + open var presentedItemURL: URL? { + return nil + } + + public var presentedItemOperationQueue: OperationQueue { + return Self.filePresenterOperationQueue + } + + open func presentedSubitemDidChange(at url: URL) { } + + open func presentedSubitemDidAppear(at url: URL) { } + +} diff --git a/Sources/TestUtils/MockKeyValueStore.swift b/Sources/TestUtils/MockKeyValueStore.swift index 7d0773f08..b13963eba 100644 --- a/Sources/TestUtils/MockKeyValueStore.swift +++ b/Sources/TestUtils/MockKeyValueStore.swift @@ -37,6 +37,10 @@ public class MockKeyValueStore: KeyValueStoring { store[defaultName] = nil } + public func clearAll() { + store.removeAll() + } + } extension MockKeyValueStore: DictionaryRepresentable { diff --git a/Tests/ConfigurationTests/ConfigurationManagerTests.swift b/Tests/ConfigurationTests/ConfigurationManagerTests.swift new file mode 100644 index 000000000..97cd49569 --- /dev/null +++ b/Tests/ConfigurationTests/ConfigurationManagerTests.swift @@ -0,0 +1,224 @@ +// +// ConfigurationManagerTests.swift +// +// Copyright © 2023 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import XCTest +import Persistence +@testable import Configuration +@testable import Networking +@testable import TestUtils + +final class MockConfigurationManager: DefaultConfigurationManager { + + var dependencyProvider: MockDependencyProvider = MockDependencyProvider() + var name: String? + + override func refreshNow(isDebug: Bool = false) async { + let configFetched = await fetchConfigDependencies(isDebug: isDebug) + if configFetched { + updateConfigDependencies() + } + } + + func fetchConfigDependencies(isDebug: Bool) async -> Bool { + do { + try await fetcher.fetch(.privacyConfiguration, isDebug: isDebug) + return true + } catch { + return false + } + } + + var onDependenciesUpdated: (() -> Void)? + func updateConfigDependencies() { + dependencyProvider.privacyConfigData = store.loadData(for: .privacyConfiguration) + dependencyProvider.privacyConfigEtag = store.loadEtag(for: .privacyConfiguration) + onDependenciesUpdated?() + } + + override var presentedItemURL: URL? { + return store.fileUrl(for: .privacyConfiguration).deletingLastPathComponent() + } + + override func presentedSubitemDidAppear(at url: URL) { + guard url == store.fileUrl(for: .privacyConfiguration) else { return } + updateConfigDependencies() + } + + override func presentedSubitemDidChange(at url: URL) { + guard url == store.fileUrl(for: .privacyConfiguration) else { return } + updateConfigDependencies() + } +} + +struct MockDependencyProvider { + var privacyConfigEtag: String? + var privacyConfigData: Data? +} + +final class ConfigurationManagerTests: XCTestCase { + + // Shared "UserDefaults" to mock app group defaults + var sharedDefaults = MockKeyValueStore() + + override func setUp() { + APIRequest.Headers.setUserAgent("") + Configuration.setURLProvider(MockConfigurationURLProvider()) + sharedDefaults.clearAll() + MockStoreWithStorage.clearTempConfigs() + } + + override func tearDown() { + MockURLProtocol.lastRequest = nil + MockURLProtocol.requestHandler = nil + } + + func makeConfigurationFetcher(store: ConfigurationStoring, + validator: ConfigurationValidating = MockValidator()) -> ConfigurationFetcher { + let testConfiguration = URLSessionConfiguration.default + testConfiguration.protocolClasses = [MockURLProtocol.self] + return ConfigurationFetcher(store: store, + validator: validator, + urlSession: URLSession(configuration: testConfiguration)) + } + + func makeConfigurationManager(name: String? = nil) -> MockConfigurationManager { + let configStore = MockStoreWithStorage(etagStorage: sharedDefaults) + let manager = MockConfigurationManager(fetcher: makeConfigurationFetcher(store: configStore), + store: configStore, + defaults: sharedDefaults) + manager.name = name + return manager + } + + func testWhenConfigIsFetchedAndStoredDependencyIsUpdated() async { + let configurationManager = makeConfigurationManager() + + let configData = Data("Privacy Config".utf8) + MockURLProtocol.requestHandler = { _ in (HTTPURLResponse.ok, configData) } + await configurationManager.refreshNow() + + XCTAssertNotNil(MockURLProtocol.lastRequest) + XCTAssertEqual(configurationManager.dependencyProvider.privacyConfigData, configData) + XCTAssertEqual(configurationManager.dependencyProvider.privacyConfigEtag, HTTPURLResponse.testEtag) + } + + func testWhenConfigIsNotModifiedThenDependencyIsNotUpdated() async { + let configurationManager = makeConfigurationManager() + + MockURLProtocol.requestHandler = { _ in (HTTPURLResponse.notModified, nil) } + await configurationManager.refreshNow() + + XCTAssertNotNil(MockURLProtocol.lastRequest) + XCTAssertNil(configurationManager.dependencyProvider.privacyConfigData) + XCTAssertNil(configurationManager.dependencyProvider.privacyConfigEtag) + } + + func testWhenManagerAIsUpdatedManagerBIsAlsoUpdated() async throws { + let managerA = makeConfigurationManager(name: "A") + let managerB = makeConfigurationManager(name: "B") + + var e: XCTestExpectation? = expectation(description: "ConfigManager B updated") + managerB.onDependenciesUpdated = { + e?.fulfill() + e = nil + } + + let configData = Data("Privacy Config".utf8) + MockURLProtocol.requestHandler = { _ in (HTTPURLResponse.ok, configData) } + await managerA.refreshNow() + await fulfillment(of: [e!], timeout: 2) + + XCTAssertNotNil(MockURLProtocol.lastRequest) + XCTAssertEqual(managerB.dependencyProvider.privacyConfigData, configData) + XCTAssertEqual(managerB.dependencyProvider.privacyConfigEtag, HTTPURLResponse.testEtag) + } + + func testWhenManagerBReceivesNewDataManagerAHasDataAfter304Response() async throws { + let managerA = makeConfigurationManager() + let managerB = makeConfigurationManager() + + var e: XCTestExpectation? = expectation(description: "ConfigManager B updated") + managerB.onDependenciesUpdated = { + e?.fulfill() + e = nil + } + + var configData = Data("Privacy Config".utf8) + MockURLProtocol.requestHandler = { _ in (HTTPURLResponse.ok, configData) } + await managerA.refreshNow() + await fulfillment(of: [e!], timeout: 2) + + XCTAssertNotNil(MockURLProtocol.lastRequest) + XCTAssertEqual(managerB.dependencyProvider.privacyConfigData, configData) + XCTAssertEqual(managerB.dependencyProvider.privacyConfigEtag, HTTPURLResponse.testEtag) + + MockURLProtocol.lastRequest = nil + e = expectation(description: "ConfigManager A updated") + managerB.onDependenciesUpdated = nil + managerA.onDependenciesUpdated = { + e?.fulfill() + e = nil + } + + configData = Data("Privacy Config 2".utf8) + MockURLProtocol.requestHandler = { _ in (HTTPURLResponse.ok, configData) } + await managerB.refreshNow() + await fulfillment(of: [e!], timeout: 2) + + XCTAssertNotNil(MockURLProtocol.lastRequest) + XCTAssertEqual(managerA.dependencyProvider.privacyConfigData, configData) + XCTAssertEqual(managerA.dependencyProvider.privacyConfigEtag, HTTPURLResponse.testEtag) + + MockURLProtocol.lastRequest = nil + MockURLProtocol.requestHandler = { _ in (HTTPURLResponse.notModified, nil) } + await managerA.refreshNow() + + XCTAssertNotNil(MockURLProtocol.lastRequest) + XCTAssertEqual(managerA.dependencyProvider.privacyConfigData, configData) + XCTAssertEqual(managerA.dependencyProvider.privacyConfigEtag, HTTPURLResponse.testEtag) + } + + func testWhenManagerBReceivesAnErrorItKeepsDataFromManagerA() async throws { + let managerA = makeConfigurationManager() + let managerB = makeConfigurationManager() + + var e: XCTestExpectation? = expectation(description: "ConfigManager B updated") + managerB.onDependenciesUpdated = { + e?.fulfill() + e = nil + } + + let configData = Data("Privacy Config".utf8) + MockURLProtocol.requestHandler = { _ in (HTTPURLResponse.ok, configData) } + await managerA.refreshNow() + await fulfillment(of: [e!], timeout: 2) + + XCTAssertNotNil(MockURLProtocol.lastRequest) + XCTAssertEqual(managerB.dependencyProvider.privacyConfigData, configData) + XCTAssertEqual(managerB.dependencyProvider.privacyConfigEtag, HTTPURLResponse.testEtag) + + MockURLProtocol.lastRequest = nil + MockURLProtocol.requestHandler = { _ in (HTTPURLResponse.internalServerError, nil) } + await managerB.refreshNow() + + XCTAssertNotNil(MockURLProtocol.lastRequest) + XCTAssertEqual(managerB.dependencyProvider.privacyConfigData, configData) + XCTAssertEqual(managerB.dependencyProvider.privacyConfigEtag, HTTPURLResponse.testEtag) + } + +} diff --git a/Tests/ConfigurationTests/Mocks/MockStore.swift b/Tests/ConfigurationTests/Mocks/MockStore.swift index da1f9b4cf..ad3feccf1 100644 --- a/Tests/ConfigurationTests/Mocks/MockStore.swift +++ b/Tests/ConfigurationTests/Mocks/MockStore.swift @@ -50,4 +50,8 @@ final class MockStore: ConfigurationStoring { try defaultSaveEtag?(etag, configuration) } + func fileUrl(for configuration: Configuration) -> URL { + return FileManager.default.temporaryDirectory.appending(configuration.rawValue) + } + } diff --git a/Tests/ConfigurationTests/Mocks/MockStoreWithStorage.swift b/Tests/ConfigurationTests/Mocks/MockStoreWithStorage.swift new file mode 100644 index 000000000..b655896ae --- /dev/null +++ b/Tests/ConfigurationTests/Mocks/MockStoreWithStorage.swift @@ -0,0 +1,91 @@ +// +// MockStoreWithStorage.swift +// +// Copyright © 2023 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +import Foundation +import Persistence +import TestUtils +@testable import Configuration + +final class MockStoreWithStorage: ConfigurationStoring { + + var etagStorage: KeyValueStoring + + init(etagStorage: KeyValueStoring) { + self.etagStorage = etagStorage + } + + func loadEtag(for configuration: Configuration) -> String? { etagStorage.object(forKey: configuration.rawValue) as? String } + func loadEmbeddedEtag(for configuration: Configuration) -> String? { nil } + + func loadData(for configuration: Configuration) -> Data? { + let file = fileUrl(for: configuration) + var data: Data? + var coordinatorError: NSError? + + NSFileCoordinator().coordinate(readingItemAt: file, error: &coordinatorError) { fileUrl in + do { + data = try Data(contentsOf: fileUrl) + } catch { + let nserror = error as NSError + + if nserror.domain != NSCocoaErrorDomain || nserror.code != NSFileReadNoSuchFileError { + fatalError("Unable to load config file: \(error.localizedDescription)") + } + } + } + + if let coordinatorError { + fatalError("Unable to read due to coordinator error: \(coordinatorError.localizedDescription)") + } + + return data + } + + func saveData(_ data: Data, for configuration: Configuration) throws { + let file = fileUrl(for: configuration) + var coordinatorError: NSError? + + NSFileCoordinator().coordinate(writingItemAt: file, options: .forReplacing, error: &coordinatorError) { fileUrl in + do { + try data.write(to: fileUrl, options: .atomic) + } catch { + fatalError("Unable to write temp configuration file: \(error.localizedDescription)") + } + } + + if let coordinatorError { + fatalError("Unable to write due to coordinator error: \(coordinatorError.localizedDescription)") + } + } + + func saveEtag(_ etag: String, for configuration: Configuration) { + etagStorage.set(etag, forKey: configuration.rawValue) + } + + func fileUrl(for configuration: Configuration) -> URL { + return FileManager.default.temporaryDirectory.appending(configuration.rawValue) + } + + static func clearTempConfigs() { + let tempStore = MockStoreWithStorage(etagStorage: MockKeyValueStore()) + for conf in Configuration.allCases { + try? FileManager.default.removeItem(at: tempStore.fileUrl(for: conf)) + } + } + +}