Skip to content

Commit

Permalink
PIR Database Migrations to Address Integrity Issues & Related Feature…
Browse files Browse the repository at this point in the history 
… Flagger (#2997)

Task/Issue URL:
https://app.asana.com/0/1206488453854252/1207806240565841/f
Tech Design URL:
https://app.asana.com/0/481882893211075/1207878066929518/f

**Description**: This PR adds a v3 PIR database migration. This
migration is intended solely to address PIR integrity issues.
  • Loading branch information
@aataraxiaa @quanganhdo
aataraxiaa authored and quanganhdo committed Jul 31, 2024
1 parent 920dcc5 commit e5ceccf
Show file tree
Hide file tree
Showing 10 changed files with 17,452 additions and 187 deletions.
3 changes: 3 additions & 0 deletions LocalPackages/DataBrokerProtection/Package.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ let package = Package(
dependencies: [
"DataBrokerProtection",
"BrowserServicesKit",
],
resources: [
.process("Resources")
]
)
]
Expand Down
4 changes: 2 additions & 2 deletions .../Sources/DataBrokerProtection/Database/DataBrokerProtectionSecureVaultErrorReporter.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ final class DataBrokerProtectionSecureVaultErrorReporter: SecureVaultReporting {
default:
pixelHandler.fire(.secureVaultInitError(error: error))
}
case .initFailed, .failedToOpenDatabase:
pixelHandler.fire(.secureVaultInitError(error: error))
case .initFailed(let cause), .failedToOpenDatabase(let cause):
pixelHandler.fire(.secureVaultInitError(error: cause))
default:
pixelHandler.fire(.secureVaultError(error: error))
}
Expand Down
601 changes: 601 additions & 0 deletions ...Sources/DataBrokerProtection/Storage/DataBrokerProtectionDatabaseMigrationsProvider.swift
View file

Large diffs are not rendered by default.

264 changes: 80 additions & 184 deletions ...rotection/Sources/DataBrokerProtection/Storage/DataBrokerProtectionDatabaseProvider.swift
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import GRDB

enum DataBrokerProtectionDatabaseErrors: Error {
case elementNotFound
case migrationFailureIntegrityCheck
}

protocol DataBrokerProtectionDatabaseProvider: SecureStorageDatabaseProvider {
Expand Down Expand Up @@ -78,196 +79,48 @@ protocol DataBrokerProtectionDatabaseProvider: SecureStorageDatabaseProvider {

final class DefaultDataBrokerProtectionDatabaseProvider: GRDBSecureStorageDatabaseProvider, DataBrokerProtectionDatabaseProvider {

typealias FeatureFlagger = DataBrokerProtectionMigrationsFeatureFlagger
typealias MigrationsProvider = DataBrokerProtectionDatabaseMigrationsProvider

public static func defaultDatabaseURL() -> URL {
return DefaultDataBrokerProtectionDatabaseProvider.databaseFilePath(directoryName: "DBP", fileName: "Vault.db", appGroupIdentifier: Bundle.main.appGroupName)
}

public init(file: URL = DefaultDataBrokerProtectionDatabaseProvider.defaultDatabaseURL(), key: Data) throws {
try super.init(file: file, key: key, writerType: .pool) { migrator in
migrator.registerMigration("v1", migrate: Self.migrateV1(database:))
migrator.registerMigration("v2", migrate: Self.migrateV2(database:))
/// Creates a DefaultDataBrokerProtectionDatabaseProvider instance
/// - Parameters:
/// - file: File URL of the database
/// - key: Key used in encryption
/// - featureFlagger: Migrations feature flagger
/// - migrationProvider: Migrations provider
/// - Returns: DefaultDataBrokerProtectionDatabaseProvider instance
public static func create<T: MigrationsProvider>(file: URL = DefaultDataBrokerProtectionDatabaseProvider.defaultDatabaseURL(),
key: Data,
featureFlagger: FeatureFlagger = DefaultDataBrokerProtectionMigrationsFeatureFlagger(),
migrationProvider: T.Type = DefaultDataBrokerProtectionDatabaseMigrationsProvider.self) throws -> DefaultDataBrokerProtectionDatabaseProvider {

if featureFlagger.isUserIn(percent: 10) {
return try DefaultDataBrokerProtectionDatabaseProvider(file: file, key: key, registerMigrationsHandler: migrationProvider.v3Migrations)
} else {
return try DefaultDataBrokerProtectionDatabaseProvider(file: file, key: key, registerMigrationsHandler: migrationProvider.v2Migrations)
}
}

static func migrateV1(database: Database) throws {
// User profile
try database.create(table: ProfileDB.databaseTableName) {
$0.autoIncrementedPrimaryKey(ProfileDB.Columns.id.name)
$0.column(ProfileDB.Columns.birthYear.name, .integer).notNull()
}

try database.create(table: NameDB.databaseTableName) {
$0.primaryKey([NameDB.Columns.first.name, NameDB.Columns.last.name, NameDB.Columns.middle.name, NameDB.Columns.profileId.name])
$0.foreignKey([NameDB.Columns.profileId.name], references: ProfileDB.databaseTableName)

$0.column(NameDB.Columns.first.name, .text).notNull()
$0.column(NameDB.Columns.last.name, .text).notNull()
$0.column(NameDB.Columns.profileId.name, .integer).notNull()
$0.column(NameDB.Columns.middle.name, .text)
$0.column(NameDB.Columns.suffix.name, .text)
}

try database.create(table: AddressDB.databaseTableName) {
$0.primaryKey([AddressDB.Columns.city.name, AddressDB.Columns.state.name, AddressDB.Columns.street.name, AddressDB.Columns.profileId.name])
$0.foreignKey([AddressDB.Columns.profileId.name], references: ProfileDB.databaseTableName)

$0.column(AddressDB.Columns.city.name, .text).notNull()
$0.column(AddressDB.Columns.state.name, .text).notNull()
$0.column(AddressDB.Columns.profileId.name, .integer).notNull()
$0.column(AddressDB.Columns.street.name, .text)
$0.column(AddressDB.Columns.zipCode.name, .text)
}

try database.create(table: PhoneDB.databaseTableName) {
$0.primaryKey([PhoneDB.Columns.phoneNumber.name, PhoneDB.Columns.profileId.name])
$0.foreignKey([PhoneDB.Columns.profileId.name], references: ProfileDB.databaseTableName)

$0.column(PhoneDB.Columns.phoneNumber.name, .text).notNull()
$0.column(PhoneDB.Columns.profileId.name, .integer).notNull()
}

// Operation and query related
try database.create(table: ProfileQueryDB.databaseTableName) {
$0.autoIncrementedPrimaryKey(ProfileQueryDB.Columns.id.name)
$0.foreignKey([ProfileQueryDB.Columns.profileId.name], references: ProfileDB.databaseTableName)

$0.column(ProfileQueryDB.Columns.profileId.name, .integer).notNull()
$0.column(ProfileQueryDB.Columns.first.name, .text).notNull()
$0.column(ProfileQueryDB.Columns.last.name, .text).notNull()
$0.column(ProfileQueryDB.Columns.middle.name, .text)
$0.column(ProfileQueryDB.Columns.suffix.name, .text)

$0.column(ProfileQueryDB.Columns.city.name, .text).notNull()
$0.column(ProfileQueryDB.Columns.state.name, .text).notNull()
$0.column(ProfileQueryDB.Columns.street.name, .text)
$0.column(ProfileQueryDB.Columns.zipCode.name, .text)

$0.column(ProfileQueryDB.Columns.phone.name, .text)
$0.column(ProfileQueryDB.Columns.birthYear.name, .integer)

$0.column(ProfileQueryDB.Columns.deprecated.name, .boolean).notNull().defaults(to: false)
}

try database.create(table: BrokerDB.databaseTableName) {
$0.autoIncrementedPrimaryKey(BrokerDB.Columns.id.name)

$0.column(BrokerDB.Columns.name.name, .text).unique().notNull()
$0.column(BrokerDB.Columns.json.name, .text).notNull()
$0.column(BrokerDB.Columns.version.name, .text).notNull()
}

try database.create(table: ScanDB.databaseTableName) {
$0.primaryKey([ScanDB.Columns.brokerId.name, ScanDB.Columns.profileQueryId.name])

$0.foreignKey([ScanDB.Columns.brokerId.name], references: BrokerDB.databaseTableName)
$0.foreignKey([ScanDB.Columns.profileQueryId.name],
references: ProfileQueryDB.databaseTableName,
onDelete: .cascade)

$0.column(ScanDB.Columns.profileQueryId.name, .integer).notNull()
$0.column(ScanDB.Columns.brokerId.name, .integer).notNull()
$0.column(ScanDB.Columns.lastRunDate.name, .datetime)
$0.column(ScanDB.Columns.preferredRunDate.name, .datetime)
}

try database.create(table: ScanHistoryEventDB.databaseTableName) {
$0.primaryKey([
ScanHistoryEventDB.Columns.brokerId.name,
ScanHistoryEventDB.Columns.profileQueryId.name,
ScanHistoryEventDB.Columns.event.name,
ScanHistoryEventDB.Columns.timestamp.name
])

$0.foreignKey([ScanDB.Columns.brokerId.name], references: BrokerDB.databaseTableName)
$0.foreignKey([ScanDB.Columns.profileQueryId.name],
references: ProfileQueryDB.databaseTableName,
onDelete: .cascade)

$0.column(ScanDB.Columns.profileQueryId.name, .integer).notNull()
$0.column(ScanDB.Columns.brokerId.name, .integer).notNull()
$0.column(ScanHistoryEventDB.Columns.event.name, .text).notNull()
$0.column(ScanHistoryEventDB.Columns.timestamp.name, .datetime).notNull()
}

try database.create(table: ExtractedProfileDB.databaseTableName) {
$0.autoIncrementedPrimaryKey(ExtractedProfileDB.Columns.id.name)

$0.foreignKey([ExtractedProfileDB.Columns.brokerId.name], references: BrokerDB.databaseTableName)
$0.foreignKey([ExtractedProfileDB.Columns.profileQueryId.name],
references: ProfileQueryDB.databaseTableName,
onDelete: .cascade)

$0.column(ExtractedProfileDB.Columns.profileQueryId.name, .integer).notNull()
$0.column(ExtractedProfileDB.Columns.brokerId.name, .integer).notNull()
$0.column(ExtractedProfileDB.Columns.profile.name, .text).notNull()
$0.column(ExtractedProfileDB.Columns.removedDate.name, .datetime)
}

try database.create(table: OptOutDB.databaseTableName) {
$0.primaryKey([
OptOutDB.Columns.profileQueryId.name,
OptOutDB.Columns.brokerId.name,
OptOutDB.Columns.extractedProfileId.name
])

$0.foreignKey([OptOutDB.Columns.brokerId.name], references: BrokerDB.databaseTableName)
$0.foreignKey([OptOutDB.Columns.profileQueryId.name],
references: ProfileQueryDB.databaseTableName,
onDelete: .cascade)

$0.foreignKey([OptOutDB.Columns.extractedProfileId.name],
references: ExtractedProfileDB.databaseTableName,
onDelete: .cascade)

$0.column(OptOutDB.Columns.profileQueryId.name, .integer).notNull()
$0.column(OptOutDB.Columns.brokerId.name, .integer).notNull()
$0.column(OptOutDB.Columns.extractedProfileId.name, .integer).notNull()
$0.column(OptOutDB.Columns.lastRunDate.name, .datetime)
$0.column(OptOutDB.Columns.preferredRunDate.name, .datetime)
}

try database.create(table: OptOutHistoryEventDB.databaseTableName) {
$0.primaryKey([
OptOutHistoryEventDB.Columns.profileQueryId.name,
OptOutHistoryEventDB.Columns.brokerId.name,
OptOutHistoryEventDB.Columns.extractedProfileId.name,
OptOutHistoryEventDB.Columns.event.name,
OptOutHistoryEventDB.Columns.timestamp.name
])

$0.foreignKey([OptOutHistoryEventDB.Columns.brokerId.name], references: BrokerDB.databaseTableName)
$0.foreignKey([OptOutHistoryEventDB.Columns.profileQueryId.name],
references: ProfileQueryDB.databaseTableName,
onDelete: .cascade)

$0.column(OptOutHistoryEventDB.Columns.profileQueryId.name, .integer).notNull()
$0.column(OptOutHistoryEventDB.Columns.brokerId.name, .integer).notNull()
$0.column(OptOutHistoryEventDB.Columns.extractedProfileId.name, .integer).notNull()
$0.column(OptOutHistoryEventDB.Columns.event.name, .text).notNull()
$0.column(OptOutHistoryEventDB.Columns.timestamp.name, .datetime).notNull()
}

try database.create(table: OptOutAttemptDB.databaseTableName) {
$0.primaryKey([OptOutAttemptDB.Columns.extractedProfileId.name])

$0.foreignKey([OptOutAttemptDB.Columns.extractedProfileId.name], references: ExtractedProfileDB.databaseTableName)

$0.column(OptOutAttemptDB.Columns.extractedProfileId.name, .integer).notNull()
$0.column(OptOutAttemptDB.Columns.dataBroker.name, .text).notNull()
$0.column(OptOutAttemptDB.Columns.attemptId.name, .text).notNull()
$0.column(OptOutAttemptDB.Columns.lastStageDate.name, .date).notNull()
$0.column(OptOutAttemptDB.Columns.startDate.name, .date).notNull()
}
public init(file: URL = DefaultDataBrokerProtectionDatabaseProvider.defaultDatabaseURL(),
key: Data,
registerMigrationsHandler: (inout DatabaseMigrator) throws -> Void) throws {
try super.init(file: file, key: key, writerType: .pool, registerMigrationsHandler: registerMigrationsHandler)
}

static func migrateV2(database: Database) throws {
try database.alter(table: BrokerDB.databaseTableName) {
$0.add(column: BrokerDB.Columns.url.name, .text)
func createFileURLInDocumentsDirectory(fileName: String) -> URL? {
let fileManager = FileManager.default
do {
let documentsDirectory = try fileManager.url(for: .documentDirectory, in: .userDomainMask, appropriateFor: nil, create: false)
let fileURL = documentsDirectory.appendingPathComponent(fileName)
return fileURL
} catch {
print("Error getting documents directory: \(error.localizedDescription)")
return nil
}
try database.execute(sql: """
UPDATE \(BrokerDB.databaseTableName) SET \(BrokerDB.Columns.url.name) = \(BrokerDB.Columns.name.name)
""")
}

func updateProfile(profile: DataBrokerProtectionProfile, mapperToDB: MapperToDB) throws -> Int64 {
Expand Down Expand Up @@ -329,23 +182,31 @@ final class DefaultDataBrokerProtectionDatabaseProvider: GRDBSecureStorageDataba
}

func deleteProfileData() throws {
try db.writeWithoutTransaction { db in
try db.execute(sql: "PRAGMA foreign_keys = OFF;")
try db.write { db in
try OptOutHistoryEventDB
.deleteAll(db)
try OptOutDB
.deleteAll(db)
try ScanHistoryEventDB
.deleteAll(db)
try ScanDB
.deleteAll(db)
try OptOutAttemptDB
.deleteAll(db)
try ExtractedProfileDB
.deleteAll(db)
try ProfileQueryDB
.deleteAll(db)
try NameDB
.deleteAll(db)
try AddressDB
.deleteAll(db)
try PhoneDB
.deleteAll(db)
try ProfileDB
.deleteAll(db)
try BrokerDB
.deleteAll(db)
try db.execute(sql: "PRAGMA foreign_keys = ON;")
try ProfileDB
.deleteAll(db)
}
}

Expand Down Expand Up @@ -668,3 +529,38 @@ final class DefaultDataBrokerProtectionDatabaseProvider: GRDBSecureStorageDataba
}
}
}

private extension DatabaseValue {

/// Returns the SQL representation of the `DatabaseValue`.
///
/// This converts the database value to a string that can be used in an SQL statement.
///
/// - Returns: A `String` representing the SQL expression of the `DatabaseValue`.
var sqlExpression: String {
switch storage {
case .null:
return "NULL"
case .int64(let int64):
return "\(int64)"
case .double(let double):
return "\(double)"
case .string(let string):
return "'\(string.replacingOccurrences(of: "'", with: "''"))'"
case .blob(let data):
return "X'\(data.hexEncodedString())'"
}
}
}

private extension Data {

/// Converts `Data` to a hexadecimal string representation.
///
/// Used to format data so it can be inserted into SQL statements.
///
/// - Returns: A `String` representing the hexadecimal encoding of the data.
func hexEncodedString() -> String {
return map { String(format: "%02hhx", $0) }.joined()
}
}
Loading

0 comments on commit e5ceccf

Please sign in to comment.