Change when/how data is synced to store

ebi-pf-team · Sep 7, 2024 · 5b622cd · 5b622cd
1 parent e007506
commit 5b622cd
Showing 1 changed file with 13 additions and 8 deletions.
diff --git a/...a/uk/ac/ebi/interpro/scan/precalc/berkeley/iprscan/CreateSiteDBFromIprscanBerkeleyDB.java b/...a/uk/ac/ebi/interpro/scan/precalc/berkeley/iprscan/CreateSiteDBFromIprscanBerkeleyDB.java
@@ -39,6 +39,8 @@ void buildDatabase(String url, String password, int fetchSize, File outputDirect
         Environment env = null;
         EntityStore store = null;
 
+        System.out.println(Utilities.getTimeAlt() + ": starting");
+
         try (Connection connection = DriverManager.getConnection(url, USER, password)) {
             // Set up the environment
             EnvironmentConfig envConfig = new EnvironmentConfig();
@@ -59,7 +61,7 @@ void buildDatabase(String url, String password, int fetchSize, File outputDirect
                 ps.setFetchSize(fetchSize);
 
                 try (ResultSet rs = ps.executeQuery()) {
-                    long matchCount = 0;
+                    int proteinCount = 0;
                     while (rs.next()) {
                         final String signatureLibraryName = rs.getString(SimpleLookupSite.COL_IDX_SIG_LIB_NAME);
                         if (rs.wasNull() || signatureLibraryName == null) continue;
@@ -123,18 +125,19 @@ void buildDatabase(String url, String password, int fetchSize, File outputDirect
                             match = new KVSequenceEntry();
                             match.setProteinMD5(proteinMD5);
                             match.addMatch(kvMatch);
-                        }
-                        matchCount++;
-                        if (matchCount % 1000000 == 0) {
-                            store.sync();
 
-                            if (matchCount % 10000000 == 0) {
-                                System.out.println(Utilities.getTimeNow() + " Stored " + matchCount + " items.");
+                            proteinCount++;
+                            if (proteinCount % 100000 == 0) {
+                                store.sync();
+
+                                if (proteinCount % 10000000 == 0) {
+                                    System.out.println(Utilities.getTimeAlt() + ": " + String.format("%,d", proteinCount) + " sequences processed");
+                                }
                             }
                         }
                     }
 
-                    System.out.println(Utilities.getTimeNow() + " Stored " + matchCount + " items.");
+                    System.out.println(Utilities.getTimeAlt() + ": " + String.format("%,d", proteinCount) + " sequences processed");
                 }
             }
 
@@ -164,6 +167,8 @@ void buildDatabase(String url, String password, int fetchSize, File outputDirect
                 }
             }
         }
+
+        System.out.println(Utilities.getTimeAlt() + ": done");
     }
 
     public static String kvValueOf(Object obj) {