cache file entries too otherwise it may be too slow

JaneliaSciComp · May 30, 2024 · e00519d · e00519d
1 parent 5538915
commit e00519d
Showing 1 changed file with 38 additions and 29 deletions.
diff --git a/colormipsearch-api/src/main/java/org/janelia/colormipsearch/mips/FileDataUtils.java b/colormipsearch-api/src/main/java/org/janelia/colormipsearch/mips/FileDataUtils.java
@@ -5,7 +5,6 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -14,19 +13,17 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.RegExUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.janelia.colormipsearch.model.FileData;
 
 public class FileDataUtils {
 
-    private static final Map<Path, Map<String, List<String>>> ARCHIVE_ENTRIES_CACHE = new HashMap<>();
+    private static final Map<Path, Map<String, List<String>>> FILE_NAMES_CACHE = new HashMap<>();
 
     public static FileData lookupVariantFileData(List<String> variantLocations, Pattern variantPattern, String sourceCDMName, String variantSuffix, Function<String, String> variantSuffixMapping) {
         if (CollectionUtils.isEmpty(variantLocations)) {
@@ -50,26 +47,17 @@ public static FileData lookupVariantFileData(List<String> variantLocations, Patt
         }
     }
 
-    private static FileData lookupVariantFileDataInDir(Path variantPath,
-                                                       Pattern variantPattern) {
-        try (Stream<Path> s = Files.find(variantPath, 1, (p, a) -> {
-            Matcher variantMatcher = variantPattern.matcher(p.getFileName().toString());
-            return variantMatcher.find();
-        })) {
-            return s.map(p -> {
-                        try {
-                            return p.toRealPath();
-                        } catch (IOException e) {
-                            throw new UncheckedIOException(e);
-                        }
-                    })
-                    .filter(Files::isRegularFile)
-                    .findFirst()
-                    .map(p -> FileData.fromString(p.toString()))
-                    .orElse(null);
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
+    private static FileData lookupVariantFileDataInDir(Path variantPath, Pattern variantPattern) {
+        Map<String, List<String>> variantDirEntries = getDirEntryNames(variantPath);
+        return variantDirEntries.entrySet().stream()
+                .filter(e -> {
+                    Matcher variantMatcher = variantPattern.matcher(e.getKey());
+                    return variantMatcher.find();
+                })
+                .flatMap(e -> e.getValue().stream())
+                .findFirst()
+                .map(FileData::fromString)
+                .orElse(null);
     }
 
     private static FileData lookupVariantFileDataInArchive(Path variantPath, Pattern variantPattern) {
@@ -86,10 +74,10 @@ private static FileData lookupVariantFileDataInArchive(Path variantPath, Pattern
     }
 
     private static Map<String, List<String>> getZipEntryNames(Path zipPath) {
-        if (ARCHIVE_ENTRIES_CACHE.get(zipPath) == null) {
+        if (FILE_NAMES_CACHE.get(zipPath) == null) {
             return cacheZipEntryNames(zipPath);
         } else {
-            return ARCHIVE_ENTRIES_CACHE.get(zipPath);
+            return FILE_NAMES_CACHE.get(zipPath);
         }
     }
 
@@ -105,7 +93,7 @@ private static Map<String, List<String>> cacheZipEntryNames(Path zipPath) {
                     .filter(ze -> !ze.isDirectory())
                     .map(ZipEntry::getName)
                     .collect(Collectors.groupingBy(zen -> Paths.get(zen).getFileName().toString(), Collectors.toList()));
-            ARCHIVE_ENTRIES_CACHE.put(zipPath, zipEntryNames);
+            FILE_NAMES_CACHE.put(zipPath, zipEntryNames);
             return zipEntryNames;
         } finally {
             try {
@@ -115,8 +103,29 @@ private static Map<String, List<String>> cacheZipEntryNames(Path zipPath) {
         }
     }
 
-    private static String createMIPName(String name, String suffix, String ext) {
-        return name + StringUtils.defaultIfEmpty(suffix, "") + ext;
+    private static Map<String, List<String>> getDirEntryNames(Path dirPath) {
+        if (FILE_NAMES_CACHE.get(dirPath) == null) {
+            return cacheDirEntryNames(dirPath);
+        } else {
+            return FILE_NAMES_CACHE.get(dirPath);
+        }
+    }
+
+    private static Map<String, List<String>> cacheDirEntryNames(Path dirPath) {
+        try (Stream<Path> s = Files.find(dirPath, 1, (p, a) -> !a.isDirectory())) {
+            Map<String, List<String>> dirEntryNames =
+                    s.map(p -> {
+                        try {
+                            return p.toRealPath().toString();
+                        } catch (IOException e) {
+                            throw new UncheckedIOException(e);
+                        }
+                    }).collect(Collectors.groupingBy(f -> Paths.get(f).getFileName().toString(), Collectors.toList()));
+            FILE_NAMES_CACHE.put(dirPath, dirEntryNames);
+            return dirEntryNames;
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
     }
 
 }