Add deduplication of captions by building a set which has superior pe…

…rformance for large datasets.
bghira · Oct 27, 2024 · f1b1073 · f1b1073
1 parent 6a7efd8
commit f1b1073
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/helpers/prompts.py b/helpers/prompts.py
@@ -462,6 +462,9 @@ def get_all_captions(
                 # allow caching of multiple captions, if returned by the backend.
                 captions.extend(caption)
 
+        # Deduplicate captions
+        captions = list(set(captions))
+
         return captions
 
     @staticmethod