De-duplicate passwords imports

DuckDuckGo.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -32,8 +32,8 @@
       "kind" : "remoteSourceControl",
       "location" : "https://github.com/duckduckgo/BrowserServicesKit",
       "state" : {
-        "revision" : "f23384018ede5aa63777b1c143e81855a16210fd",
-        "version" : "166.0.0"
+        "revision" : "5954412504b0cf294f5c0d90d7a0c8dfcd009558",
+        "version" : "167.0.0"
       }
     },
     {

DuckDuckGo/Common/Extensions/OptionalExtension.swift

@@ -18,26 +18,11 @@
 
 import Foundation
 
-protocol OptionalProtocol {
-    associatedtype Wrapped
-
-    var isNil: Bool { get }
-
-    /// instantiate a Concrete-Typed `Optional<Wrapped>.none as T` from an `AnyOptionalType`
-    /// can be used to return nil value for a maybe-optional Generic Type
-    /// usage: `(T.self as? AnyOptionalType)?.none as? T`
-    static var none: Self { get }
-}
-typealias AnyOptional = any OptionalProtocol
-typealias AnyOptionalType = any OptionalProtocol.Type
-
-extension Optional: OptionalProtocol {
-
-    var isNil: Bool {
-        if case .none = self {
-            return true
+extension Optional where Wrapped == String {
+    var isNilOrEmpty: Bool {
+        if case .some(let wrapped) = self {
+            return wrapped.isEmpty
         }
-        return false
+        return true
     }
-
 }

DuckDuckGo/DataImport/Logins/SecureVault/SecureVaultLoginImporter.swift

@@ -22,6 +22,14 @@ import SecureStorage
 
 final class SecureVaultLoginImporter: LoginImporter {
 
+    static var featureFlagger: FeatureFlagger {
+        NSApp.delegateTyped.featureFlagger
+    }
+
+    private enum ImporterError: Error {
+        case duplicate
+    }
+
     func importLogins(_ logins: [ImportedLoginCredential], progressCallback: @escaping (Int) throws -> Void) throws -> DataImport.DataTypeSummary {
         let vault = try AutofillSecureVaultFactory.makeVault(reporter: SecureVaultReporter.shared)
 
@@ -32,6 +40,8 @@ final class SecureVaultLoginImporter: LoginImporter {
         let encryptionKey = try vault.getEncryptionKey()
         let hashingSalt = try vault.getHashingSalt()
 
+        let accounts = (try? vault.accounts()) ?? .init()
+
         try vault.inDatabaseTransaction { database in
             for (idx, login) in logins.enumerated() {
                 let title = login.title
@@ -46,11 +56,22 @@ final class SecureVaultLoginImporter: LoginImporter {
                 }
 
                 do {
+                    if Self.featureFlagger.isFeatureOn(.deduplicateLoginsOnImport),
+                        let signature = try vault.encryptPassword(for: credentials, key: encryptionKey, salt: hashingSalt).account.signature {
+                        let isDuplicate = accounts.contains {
+                            $0.isDuplicateOf(accountToBeImported: account, signatureOfAccountToBeImported: signature, passwordToBeImported: login.password)
+                        }
+                        if isDuplicate {
+                            throw ImporterError.duplicate
+                        }
+                    }
                     _ = try vault.storeWebsiteCredentials(credentials, in: database, encryptedUsing: encryptionKey, hashedUsing: hashingSalt)
                     successful.append(importSummaryValue)
                 } catch {
                     if case .duplicateRecord = error as? SecureStorageError {
                         duplicates.append(importSummaryValue)
+                    } else if case .duplicate = error as? ImporterError {
+                        duplicates.append(importSummaryValue)
                     } else {
                         failed.append(importSummaryValue)
                     }
@@ -66,5 +87,27 @@ final class SecureVaultLoginImporter: LoginImporter {
 
         return .init(successful: successful.count, duplicate: duplicates.count, failed: failed.count)
     }
+}
 
+extension SecureVaultModels.WebsiteAccount {
+
+    // Deduplication rules: https://app.asana.com/0/0/1207598052765977/f
+    func isDuplicateOf(accountToBeImported: Self, signatureOfAccountToBeImported: String, passwordToBeImported: String?) -> Bool {
+        guard signature == signatureOfAccountToBeImported || passwordToBeImported.isNilOrEmpty else {
+            return false
+        }
+        guard username == accountToBeImported.username || accountToBeImported.username.isNilOrEmpty else {
+            return false
+        }
+        guard domain == accountToBeImported.domain || accountToBeImported.domain.isNilOrEmpty else {
+            return false
+        }
+        guard notes == accountToBeImported.notes || accountToBeImported.notes.isNilOrEmpty else {
+            return false
+        }
+        guard patternMatchedTitle() == accountToBeImported.patternMatchedTitle() || accountToBeImported.patternMatchedTitle().isEmpty else {
+            return false
+        }
+        return true
+    }
 }

DuckDuckGo/DataImport/View/DataImportSummaryView.swift

@@ -111,7 +111,7 @@ func bookmarksSuccessSummary(_ summary: DataImport.DataTypeSummary) -> some View
         if summary.duplicate > 0 {
             lineSeparator()
             importSummaryRow(image: .failed,
-                             text: "Duplicate Bookmarks Skipped:",
+                             text: "Duplicates Skipped:",
                              comment: "Data import summary format of how many duplicate bookmarks (%lld) were skipped during import.",
                              count: summary.duplicate)
         }
@@ -138,6 +138,13 @@ private func passwordsSuccessSummary(_ summary: DataImport.DataTypeSummary) -> s
                              comment: "Data import summary format of how many passwords (%lld) failed to import.",
                              count: summary.failed)
         }
+        if summary.duplicate > 0 {
+            lineSeparator()
+            importSummaryRow(image: .failed,
+                             text: "Duplicates Skipped: ",
+                             comment: "Data import summary format of how many passwords (%lld) were skipped due to being duplicates.",
+                             count: summary.duplicate)
+        }
     }
 }
 

DuckDuckGo/FeatureFlagging/Model/FeatureFlag.swift

@@ -26,6 +26,9 @@ public enum FeatureFlag: String {
     /// Add experimental atb parameter to SERP queries for internal users to display Privacy Reminder
     /// https://app.asana.com/0/1199230911884351/1205979030848528/f
     case appendAtbToSerpQueries
+
+    // https://app.asana.com/0/72649045549333/1207597760316574/f
+    case deduplicateLoginsOnImport
 }
 
 extension FeatureFlag: FeatureFlagSourceProviding {
@@ -37,6 +40,8 @@ extension FeatureFlag: FeatureFlagSourceProviding {
             return .internalOnly
         case .sslCertificatesBypass:
             return .remoteReleasable(.subfeature(sslCertificatesSubfeature.allowBypass))
+        case .deduplicateLoginsOnImport:
+            return .remoteReleasable(.subfeature(AutofillSubfeature.deduplicateLoginsOnImport))
         }
     }
 }

DuckDuckGo/Localizable.xcstrings

@@ -17237,6 +17237,7 @@
     },
     "Duplicate Bookmarks Skipped:" : {
       "comment" : "Data import summary format of how many duplicate bookmarks (%lld) were skipped during import.",
+      "extractionState" : "stale",
       "localizations" : {
         "de" : {
           "stringUnit" : {
@@ -17347,6 +17348,12 @@
           }
         }
       }
+    },
+    "Duplicates Skipped:" : {
+
+    },
+    "Duplicates Skipped: " : {
+
     },
     "edit" : {
       "comment" : "Edit button",

IntegrationTests/DataImport/CSVImporterIntegrationTests.swift

@@ -0,0 +1,93 @@
+//
+//  CSVImporterIntegrationTests.swift
+//
+//  Copyright © 2024 DuckDuckGo. All rights reserved.
+//
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the License.
+//  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+//  Unless required by applicable law or agreed to in writing, software
+//  distributed under the License is distributed on an "AS IS" BASIS,
+//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//  See the License for the specific language governing permissions and
+//  limitations under the License.
+//
+
+import Foundation
+import XCTest
+@testable import DuckDuckGo_Privacy_Browser
+import BrowserServicesKit
+
+final class CSVImporterIntegrationTests: XCTestCase {
+
+    override func setUpWithError() throws {
+        super.setUp()
+        try clearDB()
+        executionTimeAllowance = 10
+    }
+
+    override func tearDownWithError() throws {
+        try clearDB()
+        super.tearDown()
+    }
+
+    func clearDB() throws {
+        let vault = try AutofillSecureVaultFactory.makeVault(reporter: SecureVaultReporter.shared)
+
+        let accounts = try vault.accounts()
+        for accountID in accounts.compactMap(\.id) {
+            if let accountID = Int64(accountID) {
+                try vault.deleteWebsiteCredentialsFor(accountId: accountID)
+            }
+        }
+    }
+
+    func _testImportPasswordsPerformance() async throws {
+        let csvURL = Bundle(for: Self.self).url(forResource: "mock_login_data_large", withExtension: "csv")!
+        let csvImporter = CSVImporter(
+            fileURL: csvURL,
+            loginImporter: SecureVaultLoginImporter(),
+            defaultColumnPositions: nil
+        )
+        let importTask = csvImporter.importData(types: [.passwords])
+
+        // No baseline set, but should be no more than 0.3 seconds on an M1 Max with 32GB memory
+        measureMetrics([.wallClockTime], automaticallyStartMeasuring: false) {
+            let expectation = expectation(description: "Measure finished")
+            Task {
+                startMeasuring()
+                let result = await importTask.result
+                _ = try result.get()
+                stopMeasuring()
+                expectation.fulfill()
+            }
+            wait(for: [expectation])
+        }
+    }
+
+    // Deduplication rules: https://app.asana.com/0/0/1207598052765977/f
+    func _testImportingPasswords_deduplicatesAccordingToDefinedRules() async throws {
+        let startingDataURL = Bundle(for: Self.self).url(forResource: "login_deduplication_starting_data", withExtension: "csv")!
+        let startingDataImporter = CSVImporter(
+            fileURL: startingDataURL,
+            loginImporter: SecureVaultLoginImporter(),
+            defaultColumnPositions: nil
+        )
+        _ = await startingDataImporter.importData(types: [.passwords]).result
+
+        let testDataURL = Bundle(for: Self.self).url(forResource: "login_deduplication_test_data", withExtension: "csv")!
+        let testDataImporter = CSVImporter(
+            fileURL: testDataURL,
+            loginImporter: SecureVaultLoginImporter(),
+            defaultColumnPositions: nil
+        )
+        let importTask = testDataImporter.importData(types: [.passwords])
+        let result = await importTask.result
+        let summary = try result.get()[.passwords]?.get()
+
+        XCTAssertEqual(summary?.duplicate, 4)
+    }
+}

IntegrationTests/DataImport/login_deduplication_starting_data.csv

@@ -0,0 +1,18 @@
+title,username,password,url,notes
+Strict duplicate,isillitoe0,iL8>{M(t,http://lycos.com,"Quisque porta volutpat erat. Quisque erat eros, viverra eget, congue eget, semper rutrum, nulla. Nunc purus. Phasellus in felis. Donec semper sapien a libero. Nam dui. Proin leo odio, porttitor id, consequat in, consequat ut, nulla. Sed accumsan felis."
+Strict antithesis,aschermick1,"hS0#,JU>",https://icio.us,"Donec ut dolor. Morbi vel lectus in quam fringilla rhoncus. Mauris enim leo, rhoncus sed, vestibulum sit amet, cursus id, turpis."
+Differing usernames,karran3,rP9*0X%Xj<iBOVX~,http://angelfire.com,"Aliquam augue quam, sollicitudin vitae, consectetuer eget, rutrum at, lorem. Integer tincidunt ante vel ipsum. Praesent blandit lacinia erat. Vestibulum sed magna at nunc commodo placerat."
+Nil stored username,,rW4|)T2GagA!|NHc,http://blinklist.com,Etiam faucibus cursus urna. Ut tellus. Nulla ut erat id mauris vulputate elementum. Nullam varius. Nulla facilisi. Cras non velit nec nisi vulputate nonummy. Maecenas tincidunt lacus at velit. Vivamus vel nulla eget eros elementum pellentesque. Quisque porta volutpat erat.
+Nil imported username,ratlee5,xB0)X*hFWr,http://un.org,Nullam sit amet turpis elementum ligula vehicula consequat. Morbi a ipsum. Integer a nibh. In quis justo. Maecenas rhoncus aliquam lacus. Morbi quis tortor id nulla ultrices aliquet.
+Differing passwords,rinkin6,oK8*r5Q%O,https://psu.edu,"Donec posuere metus vitae ipsum. Aliquam non mauris. Morbi non lectus. Aliquam sit amet diam in magna bibendum imperdiet. Nullam orci pede, venenatis non, sodales sed, tincidunt eu, felis. Fusce posuere felis sed lacus. Morbi sem mauris, laoreet ut, rhoncus aliquet, pulvinar sed, nisl."
+Nil stored password,ccowan7,,http://canalblog.com,"Nulla mollis molestie lorem. Quisque ut erat. Curabitur gravida nisi at nibh. In hac habitasse platea dictumst. Aliquam augue quam, sollicitudin vitae, consectetuer eget, rutrum at, lorem. Integer tincidunt ante vel ipsum. Praesent blandit lacinia erat. Vestibulum sed magna at nunc commodo placerat. Praesent blandit."
+Nil imported password,mmonan8,oT3(a2cW$,http://symantec.com,"Phasellus in felis. Donec semper sapien a libero. Nam dui. Proin leo odio, porttitor id, consequat in, consequat ut, nulla. Sed accumsan felis. Ut at dolor quis odio consequat varius. Integer ac leo. Pellentesque ultrices mattis odio. Donec vitae nisi."
+Differing URL,hfranceschi9,iS3_XQ=}k)cvuy/,http://un.org,"Proin eu mi. Nulla ac enim. In tempor, turpis nec euismod scelerisque, quam turpis adipiscing lorem, vitae mattis nibh ligula nec sem. Duis aliquam convallis nunc. Proin at turpis a pede posuere nonummy."
+Nil stored URL,hcanepea,"kP3&*y7~26q,Veb",,In hac habitasse platea dictumst.
+Nil imported URL,ayanshonokb,rY7}IB6@,https://delicious.com,"Pellentesque at nulla. Suspendisse potenti. Cras in purus eu magna vulputate luctus. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Vivamus vestibulum sagittis sapien. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Etiam vel augue. Vestibulum rutrum rutrum neque. Aenean auctor gravida sem."
+Differing notes,nlepardc,hX4'<+Q.&<z,http://indiegogo.com,Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Mauris viverra diam vitae quam. Suspendisse potenti. Nullam porttitor lacus at turpis. Donec posuere metus vitae ipsum.
+Nil stored notes,tschalld,iK6%k&n?}OG_d,http://mysql.com,
+Nil imported notes,lhowe,gA8)oS53,http://latimes.com,"Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Proin interdum mauris non ligula pellentesque ultrices. Phasellus id sapien in sapien iaculis congue."
+Differing titles,sparncuttf,"sU1""v$c<J7<",http://statcounter.com,Curabitur convallis. Duis consequat dui nec nisi volutpat eleifend. Donec ut dolor.
+,lsambrookg,nM6)CyX&/hy,https://mapquest.com,Nil stored title
+Nil imported title,abrimbleh,iU3*~0DzuU9>0sD,https://gov.uk,Nil imported title

IntegrationTests/DataImport/login_deduplication_test_data.csv

@@ -0,0 +1,18 @@
+title,username,password,url,notes,,,,
+Strict duplicate,isillitoe0,iL8>{M(t,http://lycos.com,"Quisque porta volutpat erat. Quisque erat eros, viverra eget, congue eget, semper rutrum, nulla. Nunc purus. Phasellus in felis. Donec semper sapien a libero. Nam dui. Proin leo odio, porttitor id, consequat in, consequat ut, nulla. Sed accumsan felis.",,,,
+Complete antithesis,abrimbleh,iU3*~0DzuU9>0sD,https://icio.us,"Quisque erat eros, viverra eget, congue eget, semper rutrum, nulla. Nunc purus. Phasellus in felis. Donec semper sapien a libero. Nam dui. Proin leo odio, porttitor id, consequat in, consequat ut, nulla. Sed accumsan felis. Ut at dolor quis odio consequat varius. Integer ac leo. Pellentesque ultrices mattis odio.Hatity",caizic2,yN1{Yv%nkJ=,http://aboutads.info/diam/nam/tristique/tortor.png?ac=sapien&nulla=a,"Duis consequat dui nec nisi volutpat eleifend. Donec ut dolor. Morbi vel lectus in quam fringilla rhoncus. Mauris enim leo, rhoncus sed, vestibulum sit amet, cursus id, turpis. Integer aliquet, massa id lobortis convallis, tortor risus dapibus augue, vel accumsan tellus nisi eu orci. Mauris lacinia sapien quis libero. Nullam sit amet turpis elementum ligula vehicula consequat."
+Differing usernames,wblindmann1,rP9*0X%Xj<iBOVX~,http://angelfire.com,"Aliquam augue quam, sollicitudin vitae, consectetuer eget, rutrum at, lorem. Integer tincidunt ante vel ipsum. Praesent blandit lacinia erat. Vestibulum sed magna at nunc commodo placerat.",,,,
+Nil stored username,dprescott4,rW4|)T2GagA!|NHc,http://blinklist.com,Etiam faucibus cursus urna. Ut tellus. Nulla ut erat id mauris vulputate elementum. Nullam varius. Nulla facilisi. Cras non velit nec nisi vulputate nonummy. Maecenas tincidunt lacus at velit. Vivamus vel nulla eget eros elementum pellentesque. Quisque porta volutpat erat.,,,,
+Nil imported username,,xB0)X*hFWr,http://un.org,Nullam sit amet turpis elementum ligula vehicula consequat. Morbi a ipsum. Integer a nibh. In quis justo. Maecenas rhoncus aliquam lacus. Morbi quis tortor id nulla ultrices aliquet.,,,,
+Differing passwords,rinkin6,yT7/=or7WhIX<,https://psu.edu,"Donec posuere metus vitae ipsum. Aliquam non mauris. Morbi non lectus. Aliquam sit amet diam in magna bibendum imperdiet. Nullam orci pede, venenatis non, sodales sed, tincidunt eu, felis. Fusce posuere felis sed lacus. Morbi sem mauris, laoreet ut, rhoncus aliquet, pulvinar sed, nisl.",,,,
+Nil stored password,ccowan7,"qZ0""2*S?uO/k{",http://canalblog.com,"Nulla mollis molestie lorem. Quisque ut erat. Curabitur gravida nisi at nibh. In hac habitasse platea dictumst. Aliquam augue quam, sollicitudin vitae, consectetuer eget, rutrum at, lorem. Integer tincidunt ante vel ipsum. Praesent blandit lacinia erat. Vestibulum sed magna at nunc commodo placerat. Praesent blandit.",,,,
+Nil imported password,mmonan8,,http://symantec.com,"Phasellus in felis. Donec semper sapien a libero. Nam dui. Proin leo odio, porttitor id, consequat in, consequat ut, nulla. Sed accumsan felis. Ut at dolor quis odio consequat varius. Integer ac leo. Pellentesque ultrices mattis odio. Donec vitae nisi.",,,,
+Differing URL,hfranceschi9,iS3_XQ=}k)cvuy/,https://sogou.com,"Proin eu mi. Nulla ac enim. In tempor, turpis nec euismod scelerisque, quam turpis adipiscing lorem, vitae mattis nibh ligula nec sem. Duis aliquam convallis nunc. Proin at turpis a pede posuere nonummy.",,,,
+Nil stored URL,hcanepea,"kP3&*y7~26q,Veb",https://icq.com,In hac habitasse platea dictumst.,,,,
+Nil imported URL,ayanshonokb,rY7}IB6@,,"Pellentesque at nulla. Suspendisse potenti. Cras in purus eu magna vulputate luctus. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Vivamus vestibulum sagittis sapien. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Etiam vel augue. Vestibulum rutrum rutrum neque. Aenean auctor gravida sem.",,,,
+Differing notes,nlepardc,hX4'<+Q.&<z,http://indiegogo.com,"Integer aliquet, massa id lobortis convallis, tortor risus dapibus augue, vel accumsan tellus nisi eu orci. Mauris lacinia sapien quis libero. Nullam sit amet turpis elementum ligula vehicula consequat. Morbi a ipsum.",,,,
+Nil stored notes,tschalld,iK6%k&n?}OG_d,http://mysql.com,"Nullam varius. Nulla facilisi. Cras non velit nec nisi vulputate nonummy. Maecenas tincidunt lacus at velit. Vivamus vel nulla eget eros elementum pellentesque. Quisque porta volutpat erat. Quisque erat eros, viverra eget, congue eget, semper rutrum, nulla. Nunc purus.",,,,
+Nil imported notes,lhowe,gA8)oS53,http://latimes.com,,,,,
+Varying titles,sparncuttf,"sU1""v$c<J7<",http://statcounter.com,Curabitur convallis. Duis consequat dui nec nisi volutpat eleifend. Donec ut dolor.,,,,
+Nil stored title,lsambrookg,nM6)CyX&/hy,https://mapquest.com,Nil stored title,,,,
+,abrimbleh,iU3*~0DzuU9>0sD,https://gov.uk,Nil imported title,,,,