From bf77341e6023cdc98c119b862ad7594b21beaaaa Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Fri, 15 Nov 2024 08:37:50 -0800 Subject: [PATCH 1/3] ignore after retries --- .../pds/registry/common/util/file/FileDownloader.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/util/file/FileDownloader.java b/src/main/java/gov/nasa/pds/registry/common/util/file/FileDownloader.java index d3eec9b..16b6316 100644 --- a/src/main/java/gov/nasa/pds/registry/common/util/file/FileDownloader.java +++ b/src/main/java/gov/nasa/pds/registry/common/util/file/FileDownloader.java @@ -3,7 +3,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.InputStream; - +import java.util.ArrayList; import javax.net.ssl.SSLContext; import org.apache.http.HttpEntity; @@ -37,6 +37,7 @@ */ public class FileDownloader { + final private static ArrayList ignore = new ArrayList(); private Logger log; private int numRetries = 3; @@ -66,7 +67,7 @@ public void download(String fromUrl, File toFile) throws Exception { int count = 0; - while(true) + while(!ignore.contains(fromUrl)) { try { @@ -84,7 +85,8 @@ public void download(String fromUrl, File toFile) throws Exception } else { - throw new Exception("Could not download " + fromUrl); + ignore.add(fromUrl); + throw new Exception("Could not download " + fromUrl); } } } From e25982321ae36ef24c10040f1d0ae23517666795 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Fri, 15 Nov 2024 08:40:20 -0800 Subject: [PATCH 2/3] improve message to notify user of ignoring this LDD --- .../gov/nasa/pds/registry/common/util/file/FileDownloader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/util/file/FileDownloader.java b/src/main/java/gov/nasa/pds/registry/common/util/file/FileDownloader.java index 16b6316..bf6af34 100644 --- a/src/main/java/gov/nasa/pds/registry/common/util/file/FileDownloader.java +++ b/src/main/java/gov/nasa/pds/registry/common/util/file/FileDownloader.java @@ -86,7 +86,7 @@ public void download(String fromUrl, File toFile) throws Exception else { ignore.add(fromUrl); - throw new Exception("Could not download " + fromUrl); + throw new Exception("Could not download " + fromUrl + " and will not try again in this running instance."); } } } From 859c01ead7ac56c22533954bd1cf4632daa2fee7 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Fri, 15 Nov 2024 08:50:55 -0800 Subject: [PATCH 3/3] handle the case of ignoring futher up the call stack --- .../nasa/pds/registry/common/es/service/SchemaUpdater.java | 5 +++-- .../nasa/pds/registry/common/util/file/FileDownloader.java | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java b/src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java index 447f31e..9e27d91 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java @@ -133,8 +133,9 @@ private void updateLdd(String uri, String prefix) throws Exception try { - fileDownloader.download(jsonUrl, lddFile); - lddLoader.load(lddFile, schemaFileName, prefix); + if (fileDownloader.download(jsonUrl, lddFile)) { + lddLoader.load(lddFile, schemaFileName, prefix); + } } catch(Exception ex) { diff --git a/src/main/java/gov/nasa/pds/registry/common/util/file/FileDownloader.java b/src/main/java/gov/nasa/pds/registry/common/util/file/FileDownloader.java index bf6af34..08ca82c 100644 --- a/src/main/java/gov/nasa/pds/registry/common/util/file/FileDownloader.java +++ b/src/main/java/gov/nasa/pds/registry/common/util/file/FileDownloader.java @@ -63,7 +63,7 @@ public FileDownloader(boolean sslTrustAll) throws Exception * @param toFile Save to this file * @throws Exception an exception */ - public void download(String fromUrl, File toFile) throws Exception + public boolean download(String fromUrl, File toFile) throws Exception { int count = 0; @@ -73,7 +73,6 @@ public void download(String fromUrl, File toFile) throws Exception { count++; downloadOnce(fromUrl, toFile); - return; } catch(Exception ex) { @@ -90,6 +89,7 @@ public void download(String fromUrl, File toFile) throws Exception } } } + return ignore.contains(fromUrl); }