Add attemptCount and maxAttempts to broker config (#3533)

Task/Issue URL:
https://app.asana.com/0/72649045549333/1208700893044577/f
Tech Design URL:
https://app.asana.com/0/481882893211075/1208663928051302/f
CC:

**Description**:

This implements the retry mechanism update portion of the TD. 

**Steps to test this PR**:
1. Run the DBP flow
2. Open the DB browser > OptOutDB
3. You should see rows with successful submitted date having non-zero
attemptCount. The rest will be 0.

<!--
Tagging instructions
If this PR isn't ready to be merged for whatever reason it should be
marked with the `DO NOT MERGE` label (particularly if it's a draft)
If it's pending Product Review/PFR, please add the `Pending Product
Review` label.

If at any point it isn't actively being worked on/ready for
review/otherwise moving forward (besides the above PR/PFR exception)
strongly consider closing it (or not opening it in the first place). If
you decide not to close it, make sure it's labelled to make it clear the
PRs state and comment with more information.
-->

**Definition of Done**:

* [ ] Does this PR satisfy our [Definition of
Done](https://app.asana.com/0/1202500774821704/1207634633537039/f)?

---
###### Internal references:
[Pull Request Review
Checklist](https://app.asana.com/0/1202500774821704/1203764234894239/f)
[Software Engineering
Expectations](https://app.asana.com/0/59792373528535/199064865822552)
[Technical Design
Template](https://app.asana.com/0/59792373528535/184709971311943)
[Pull Request
Documentation](https://app.asana.com/0/1202500774821704/1204012835277482/f)

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Database/DataBrokerProtectionDatabase.swift

@@ -38,6 +38,8 @@ public protocol DataBrokerProtectionRepository {
     func updatePreferredRunDate(_ date: Date?, brokerId: Int64, profileQueryId: Int64, extractedProfileId: Int64) throws
     func updateLastRunDate(_ date: Date?, brokerId: Int64, profileQueryId: Int64) throws
     func updateLastRunDate(_ date: Date?, brokerId: Int64, profileQueryId: Int64, extractedProfileId: Int64) throws
+    func updateAttemptCount(_ count: Int64, brokerId: Int64, profileQueryId: Int64, extractedProfileId: Int64) throws
+    func incrementAttemptCount(brokerId: Int64, profileQueryId: Int64, extractedProfileId: Int64) throws
     func updateSubmittedSuccessfullyDate(_ date: Date?,
                                          forBrokerId brokerId: Int64,
                                          profileQueryId: Int64,
@@ -62,6 +64,7 @@ public protocol DataBrokerProtectionRepository {
     func fetchOptOutHistoryEvents(brokerId: Int64, profileQueryId: Int64, extractedProfileId: Int64) throws -> [HistoryEvent]
     func hasMatches() throws -> Bool
 
+    func fetchAllAttempts() throws -> [AttemptInformation]
     func fetchAttemptInformation(for extractedProfileId: Int64) throws -> AttemptInformation?
     func addAttempt(extractedProfileId: Int64, attemptUUID: UUID, dataBroker: String, lastStageDate: Date, startTime: Date) throws
 }
@@ -238,6 +241,37 @@ final class DataBrokerProtectionDatabase: DataBrokerProtectionRepository {
         }
     }
 
+    func updateAttemptCount(_ count: Int64, brokerId: Int64, profileQueryId: Int64, extractedProfileId: Int64) throws {
+        do {
+            let vault = try self.vault ?? DataBrokerProtectionSecureVaultFactory.makeVault(reporter: secureVaultErrorReporter)
+            try vault.updateAttemptCount(
+                count,
+                brokerId: brokerId,
+                profileQueryId: profileQueryId,
+                extractedProfileId: extractedProfileId
+            )
+        } catch {
+            Logger.dataBrokerProtection.error("Database error: updateAttemptCount, error: \(error.localizedDescription, privacy: .public)")
+            pixelHandler.fire(.generalError(error: error, functionOccurredIn: "DataBrokerProtectionDatabase.updateAttemptCount"))
+            throw error
+        }
+    }
+
+    func incrementAttemptCount(brokerId: Int64, profileQueryId: Int64, extractedProfileId: Int64) throws {
+        do {
+            let vault = try self.vault ?? DataBrokerProtectionSecureVaultFactory.makeVault(reporter: secureVaultErrorReporter)
+            try vault.incrementAttemptCount(
+                brokerId: brokerId,
+                profileQueryId: profileQueryId,
+                extractedProfileId: extractedProfileId
+            )
+        } catch {
+            Logger.dataBrokerProtection.error("Database error: incrementAttemptCount, error: \(error.localizedDescription, privacy: .public)")
+            pixelHandler.fire(.generalError(error: error, functionOccurredIn: "DataBrokerProtectionDatabase.incrementAttemptCount"))
+            throw error
+        }
+    }
+
     func updateSubmittedSuccessfullyDate(_ date: Date?,
                                          forBrokerId brokerId: Int64,
                                          profileQueryId: Int64,
@@ -388,6 +422,7 @@ final class DataBrokerProtectionDatabase: DataBrokerProtectionRepository {
                            createdDate: optOut.createdDate,
                            lastRunDate: optOut.lastRunDate,
                            preferredRunDate: optOut.preferredRunDate,
+                           attemptCount: optOut.attemptCount,
                            submittedSuccessfullyDate: optOut.submittedSuccessfullyDate,
                            sevenDaysConfirmationPixelFired: optOut.sevenDaysConfirmationPixelFired,
                            fourteenDaysConfirmationPixelFired: optOut.fourteenDaysConfirmationPixelFired,
@@ -453,6 +488,17 @@ final class DataBrokerProtectionDatabase: DataBrokerProtectionRepository {
         }
     }
 
+    func fetchAllAttempts() throws -> [AttemptInformation] {
+        do {
+            let vault = try self.vault ?? DataBrokerProtectionSecureVaultFactory.makeVault(reporter: secureVaultErrorReporter)
+            return try vault.fetchAllAttempts()
+        } catch {
+            Logger.dataBrokerProtection.error("Database error: fetchAllAttempts, error: \(error.localizedDescription, privacy: .public)")
+            pixelHandler.fire(.generalError(error: error, functionOccurredIn: "DataBrokerProtectionDatabase.fetchAllAttempts"))
+            throw error
+        }
+    }
+
     func fetchAttemptInformation(for extractedProfileId: Int64) throws -> AttemptInformation? {
         do {
             let vault = try self.vault ?? DataBrokerProtectionSecureVaultFactory.makeVault(reporter: secureVaultErrorReporter)

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/DebugUI/DataBaseBrowser/DataBrokerDatabaseBrowserViewModel.swift

@@ -48,7 +48,8 @@ final class DataBrokerDatabaseBrowserViewModel: ObservableObject {
         guard let dataManager = self.dataManager else { return }
 
         Task {
-            guard let data = try? dataManager.fetchBrokerProfileQueryData(ignoresCache: true) else {
+            guard let data = try? dataManager.fetchBrokerProfileQueryData(ignoresCache: true),
+                  let attempts = try? dataManager.fetchAllOptOutAttempts() else {
                 assertionFailure("DataManager error during DataBrokerDatavaseBrowserViewModel.updateTables")
                 return
             }
@@ -68,9 +69,10 @@ final class DataBrokerDatabaseBrowserViewModel: ObservableObject {
             let optOutsTable = createTable(using: optOutJobs, tableName: "OptOutOperation")
             let extractedProfilesTable = createTable(using: extractedProfiles, tableName: "ExtractedProfile")
             let eventsTable = createTable(using: events.sorted(by: { $0.date < $1.date }), tableName: "Events")
+            let attemptsTable = createTable(using: attempts.sorted(by: <), tableName: "OptOutAttempts")
 
             DispatchQueue.main.async {
-                self.tables = [brokersTable, profileQueriesTable, scansTable, optOutsTable, extractedProfilesTable, eventsTable]
+                self.tables = [brokersTable, profileQueriesTable, scansTable, optOutsTable, extractedProfilesTable, eventsTable, attemptsTable]
             }
         }
  }
@@ -136,3 +138,25 @@ struct DataBrokerDatabaseBrowserData {
     }
 
 }
+
+extension DataBrokerProtectionDataManager {
+    func fetchAllOptOutAttempts() throws -> [AttemptInformation] {
+        try database.fetchAllAttempts()
+    }
+}
+
+extension AttemptInformation: Comparable {
+    public static func < (lhs: AttemptInformation, rhs: AttemptInformation) -> Bool {
+        if lhs.extractedProfileId != rhs.extractedProfileId {
+            return lhs.extractedProfileId < rhs.extractedProfileId
+        } else if lhs.dataBroker != rhs.dataBroker {
+            return lhs.dataBroker < rhs.dataBroker
+        } else {
+            return lhs.startDate < rhs.startDate
+        }
+    }
+
+    public static func == (lhs: AttemptInformation, rhs: AttemptInformation) -> Bool {
+        lhs.attemptId == rhs.attemptId
+    }
+}

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/BrokerJobData.swift

@@ -74,6 +74,7 @@ public struct OptOutJobData: BrokerJobData, Sendable {
     let preferredRunDate: Date?
     let historyEvents: [HistoryEvent]
     let lastRunDate: Date?
+    let attemptCount: Int64
 
     // This was added in a later DB migration (V4), so will be nil for older entries submitted before the migration
     let submittedSuccessfullyDate: Date?
@@ -89,6 +90,7 @@ public struct OptOutJobData: BrokerJobData, Sendable {
          preferredRunDate: Date? = nil,
          historyEvents: [HistoryEvent],
          lastRunDate: Date? = nil,
+         attemptCount: Int64,
          submittedSuccessfullyDate: Date? = nil,
          extractedProfile: ExtractedProfile,
          sevenDaysConfirmationPixelFired: Bool = false,
@@ -100,6 +102,7 @@ public struct OptOutJobData: BrokerJobData, Sendable {
         self.preferredRunDate = preferredRunDate
         self.historyEvents = historyEvents
         self.lastRunDate = lastRunDate
+        self.attemptCount = attemptCount
         self.submittedSuccessfullyDate = submittedSuccessfullyDate
         self.extractedProfile = extractedProfile
         self.sevenDaysConfirmationPixelFired = sevenDaysConfirmationPixelFired

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift

@@ -24,6 +24,7 @@ struct DataBrokerScheduleConfig: Codable {
     let retryError: Int
     let confirmOptOutScan: Int
     let maintenanceScan: Int
+    let maxAttempts: Int
 }
 
 extension Int {

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift

@@ -176,6 +176,7 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager {
                                                           createdDate: Date(),
                                                           preferredRunDate: preferredRunOperation,
                                                           historyEvents: [HistoryEvent](),
+                                                          attemptCount: 0,
                                                           submittedSuccessfullyDate: nil,
                                                           extractedProfile: extractedProfile,
                                                           sevenDaysConfirmationPixelFired: false,
@@ -352,11 +353,17 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager {
                                                         profileQueryId: profileQueryId)
 
             try database.addAttempt(extractedProfileId: extractedProfileId,
-                                attemptUUID: stageDurationCalculator.attemptId,
-                                dataBroker: stageDurationCalculator.dataBroker,
-                                lastStageDate: stageDurationCalculator.lastStateTime,
-                                startTime: stageDurationCalculator.startTime)
+                                    attemptUUID: stageDurationCalculator.attemptId,
+                                    dataBroker: stageDurationCalculator.dataBroker,
+                                    lastStageDate: stageDurationCalculator.lastStateTime,
+                                    startTime: stageDurationCalculator.startTime)
             try database.add(.init(extractedProfileId: extractedProfileId, brokerId: brokerId, profileQueryId: profileQueryId, type: .optOutRequested))
+            try incrementAttemptCountIfNeeded(
+                database: database,
+                brokerId: brokerId,
+                profileQueryId: profileQueryId,
+                extractedProfileId: extractedProfileId
+            )
         } catch {
             let tries = try? retriesCalculatorUseCase.calculateForOptOut(database: database, brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId)
             stageDurationCalculator.fireOptOutFailure(tries: tries ?? -1)
@@ -389,6 +396,18 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager {
                                                      schedulingConfig: schedulingConfig)
         }
 
+    private func incrementAttemptCountIfNeeded(database: DataBrokerProtectionRepository,
+                                               brokerId: Int64,
+                                               profileQueryId: Int64,
+                                               extractedProfileId: Int64) throws {
+        guard let events = try? database.fetchOptOutHistoryEvents(brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId),
+              events.max(by: { $0.date < $1.date })?.type == .optOutRequested else {
+            return
+        }
+
+        try database.incrementAttemptCount(brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId)
+    }
+
     private func handleOperationError(origin: OperationPreferredDateUpdaterOrigin,
                                       brokerId: Int64,
                                       profileQueryId: Int64,

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProtectionBrokerUpdater.swift

@@ -202,6 +202,18 @@ public struct DefaultDataBrokerProtectionBrokerUpdater: DataBrokerProtectionBrok
             guard let savedBrokerId = savedBroker.id else { return }
 
             try vault.update(broker, with: savedBrokerId)
+            try updateAttemptCount(broker)
+        }
+    }
+
+    private func updateAttemptCount(_ broker: DataBroker) throws {
+        guard broker.type == .parent, let brokerId = broker.id else { return }
+
+        let optOutJobs = try vault.fetchOptOuts(brokerId: brokerId)
+        for optOutJob in optOutJobs {
+            if let extractedProfileId = optOutJob.extractedProfile.id {
+                try vault.updateAttemptCount(0, brokerId: brokerId, profileQueryId: optOutJob.profileQueryId, extractedProfileId: extractedProfileId)
+            }
         }
     }
 

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift

@@ -61,6 +61,7 @@ struct OperationPreferredDateCalculator {
                                 historyEvents: [HistoryEvent],
                                 extractedProfileID: Int64?,
                                 schedulingConfig: DataBrokerScheduleConfig,
+                                attemptCount: Int64?,
                                 date: DateProtocol = SystemDate()) throws -> Date? {
         guard let lastEvent = historyEvents.last else {
             throw DataBrokerProtectionError.cantCalculatePreferredRunDate
@@ -70,7 +71,8 @@ struct OperationPreferredDateCalculator {
         case .matchesFound, .reAppearence:
             if let extractedProfileID = extractedProfileID, shouldScheduleNewOptOut(events: historyEvents,
                                                                                     extractedProfileId: extractedProfileID,
-                                                                                    schedulingConfig: schedulingConfig) {
+                                                                                    schedulingConfig: schedulingConfig,
+                                                                                    attemptCount: attemptCount) {
                 return date.now
             } else {
                 return currentPreferredRunDate
@@ -84,10 +86,18 @@ struct OperationPreferredDateCalculator {
         }
     }
 
-    // If the time elapsed since the last profile removal exceeds the current date plus maintenance period (expired), we should proceed with scheduling a new opt-out request as the broker has failed to honor the previous one.
+    // If the time elapsed since the last profile removal exceeds the current date plus maintenance period (expired),
+    // and the number of attempts is still fewer than the configurable limit,
+    // we should proceed with scheduling a new opt-out request as the broker has failed to honor the previous one.
     private func shouldScheduleNewOptOut(events: [HistoryEvent],
                                          extractedProfileId: Int64,
-                                         schedulingConfig: DataBrokerScheduleConfig) -> Bool {
+                                         schedulingConfig: DataBrokerScheduleConfig,
+                                         attemptCount: Int64?) -> Bool {
+        let currentAttempt = attemptCount ?? 0
+        if schedulingConfig.maxAttempts != -1, currentAttempt >= schedulingConfig.maxAttempts {
+            return false
+        }
+
         guard let lastRemovalEvent = events.last(where: { $0.type == .optOutRequested && $0.extractedProfileId == extractedProfileId }) else {
             return false
         }

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateUpdater.swift

@@ -128,7 +128,8 @@ struct OperationPreferredDateUpdaterUseCase: OperationPreferredDateUpdater {
         var newOptOutPreferredDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: currentOptOutPreferredRunDate,
                                                                            historyEvents: brokerProfileQuery.events,
                                                                            extractedProfileID: extractedProfileId,
-                                                                           schedulingConfig: schedulingConfig)
+                                                                           schedulingConfig: schedulingConfig,
+                                                                           attemptCount: optOutJob?.attemptCount)
 
         if let newDate = newOptOutPreferredDate, origin == .scan {
             newOptOutPreferredDate = returnMostRecentDate(currentOptOutPreferredRunDate, newDate)

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/advancedbackgroundchecks.com.json

@@ -1,7 +1,7 @@
 {
   "name": "AdvancedBackgroundChecks",
   "url": "advancedbackgroundchecks.com",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "parent": "peoplefinders.com",
   "addedDatetime": 1678060800000,
   "optOutUrl": "https://www.advancedbackgroundchecks.com/removal",
@@ -64,6 +64,7 @@
   "schedulingConfig": {
     "retryError": 48,
     "confirmOptOutScan": 72,
-    "maintenanceScan": 240
+    "maintenanceScan": 120,
+    "maxAttempts": -1
   }
 }

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json

@@ -1,7 +1,7 @@
 {
   "name": "backgroundcheck.run",
   "url": "backgroundcheck.run",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "parent": "verecor.com",
   "addedDatetime": 1677736800000,
   "optOutUrl": "https://backgroundcheck.run/ng/control/privacy",
@@ -58,6 +58,7 @@
   "schedulingConfig": {
     "retryError": 48,
     "confirmOptOutScan": 72,
-    "maintenanceScan": 240
+    "maintenanceScan": 120,
+    "maxAttempts": -1
   }
 }

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/centeda.com.json

@@ -1,7 +1,7 @@
 {
   "name": "Centeda",
   "url": "centeda.com",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "parent": "verecor.com",
   "addedDatetime": 1677736800000,
   "optOutUrl": "https://centeda.com/ng/control/privacy",
@@ -66,6 +66,7 @@
   "schedulingConfig": {
     "retryError": 48,
     "confirmOptOutScan": 72,
-    "maintenanceScan": 240
+    "maintenanceScan": 120,
+    "maxAttempts": -1
   }
 }

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json

@@ -1,7 +1,7 @@
 {
   "name": "Clubset",
   "url": "clubset.com",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "parent": "verecor.com",
   "addedDatetime": 1702965600000,
   "optOutUrl": "https://clubset.com/private/control/privacy",
@@ -70,6 +70,7 @@
   "schedulingConfig": {
     "retryError": 48,
     "confirmOptOutScan": 72,
-    "maintenanceScan": 240
+    "maintenanceScan": 120,
+    "maxAttempts": -1
   }
 }

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clustrmaps.com.json

@@ -1,7 +1,7 @@
 {
   "name": "ClustrMaps",
   "url": "clustrmaps.com",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "parent": "neighbor.report",
   "addedDatetime": 1692594000000,
   "optOutUrl": "https://clustrmaps.com/bl/opt-out",
@@ -58,6 +58,7 @@
   "schedulingConfig": {
     "retryError": 48,
     "confirmOptOutScan": 72,
-    "maintenanceScan": 240
+    "maintenanceScan": 120,
+    "maxAttempts": -1
   }
 }

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/councilon.com.json

@@ -1,7 +1,7 @@
 {
   "name": "Councilon",
   "url": "councilon.com",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "parent": "verecor.com",
   "addedDatetime": 1702965600000,
   "optOutUrl": "https://councilon.com/ex/control/privacy",
@@ -70,6 +70,7 @@
   "schedulingConfig": {
     "retryError": 48,
     "confirmOptOutScan": 72,
-    "maintenanceScan": 240
+    "maintenanceScan": 120,
+    "maxAttempts": -1
   }
 }